From 78fc3c9bc57eb904ff6a056dff6e65d12d1d0c77 Mon Sep 17 00:00:00 2001 From: Tino Reichardt Date: Sun, 21 Oct 2018 14:23:28 +0200 Subject: [PATCH] Update to 7-Zip Version 18.05 --- Asm/x86/7zAsm.asm | 44 +- Asm/x86/LzmaDecOpt.asm | 1258 +++++++++ C/7zVersion.h | 10 +- C/Alloc.c | 354 ++- C/Alloc.h | 16 +- C/Bcj2.c | 10 +- C/Bcj2Enc.c | 3 +- C/CpuArch.h | 4 +- C/Lzma2Dec.c | 206 +- C/Lzma2Dec.h | 60 +- C/Lzma2DecMt.c | 1082 ++++++++ C/Lzma2DecMt.h | 79 + C/Lzma2Enc.c | 23 +- C/LzmaDec.c | 379 +-- C/LzmaDec.h | 41 +- C/LzmaEnc.c | 2251 ++++++++++------- C/MtCoder.c | 89 +- C/MtCoder.h | 28 +- C/MtDec.c | 1137 +++++++++ C/MtDec.h | 201 ++ C/Util/7z/7zMain.c | 27 +- C/Util/7zipInstall/makefile | 1 + C/Util/7zipUninstall/7zipUninstall.c | 6 +- C/Util/7zipUninstall/makefile | 1 + C/Util/SfxSetup/makefile | 1 + C/Util/SfxSetup/makefile_con | 1 + C/Xz.h | 190 +- C/XzDec.c | 2123 ++++++++++++++-- C/XzEnc.c | 31 +- C/XzIn.c | 6 +- CPP/7zip/Archive/7z/7zCompressionMode.h | 2 + CPP/7zip/Archive/7z/7zDecode.cpp | 46 +- CPP/7zip/Archive/7z/7zDecode.h | 4 +- CPP/7zip/Archive/7z/7zEncode.cpp | 11 +- CPP/7zip/Archive/7z/7zExtract.cpp | 10 +- CPP/7zip/Archive/7z/7zHandler.cpp | 17 +- CPP/7zip/Archive/7z/7zHandler.h | 39 +- CPP/7zip/Archive/7z/7zHandlerOut.cpp | 28 +- CPP/7zip/Archive/7z/7zIn.cpp | 4 +- CPP/7zip/Archive/7z/7zUpdate.cpp | 11 +- CPP/7zip/Archive/Cab/CabHandler.cpp | 53 +- CPP/7zip/Archive/Common/HandlerOut.cpp | 150 +- CPP/7zip/Archive/Common/HandlerOut.h | 65 +- CPP/7zip/Archive/DmgHandler.cpp | 104 +- CPP/7zip/Archive/ElfHandler.cpp | 1 + CPP/7zip/Archive/HfsHandler.cpp | 110 +- CPP/7zip/Archive/IArchive.h | 10 + CPP/7zip/Archive/Iso/IsoIn.cpp | 2 +- CPP/7zip/Archive/PeHandler.cpp | 325 ++- CPP/7zip/Archive/Rar/Rar5Handler.cpp | 23 +- CPP/7zip/Archive/Rar/Rar5Handler.h | 2 +- CPP/7zip/Archive/Rar/RarHandler.cpp | 12 +- CPP/7zip/Archive/Rar/RarHandler.h | 2 +- CPP/7zip/Archive/SquashfsHandler.cpp | 7 +- CPP/7zip/Archive/Tar/TarIn.cpp | 4 +- CPP/7zip/Archive/XzHandler.cpp | 249 +- CPP/7zip/Archive/Zip/ZipAddCommon.cpp | 2 +- CPP/7zip/Archive/Zip/ZipHandler.cpp | 153 +- CPP/7zip/Archive/Zip/ZipHandler.h | 7 + CPP/7zip/Archive/Zip/ZipHandlerOut.cpp | 31 +- CPP/7zip/Archive/Zip/ZipHeader.h | 1 + CPP/7zip/Archive/Zip/ZipIn.cpp | 29 +- CPP/7zip/Archive/Zip/ZipIn.h | 7 +- CPP/7zip/Archive/Zip/ZipItem.cpp | 6 + CPP/7zip/Archive/Zip/ZipItem.h | 7 + CPP/7zip/Archive/Zip/ZipUpdate.cpp | 26 +- CPP/7zip/Archive/Zip/ZipUpdate.h | 10 +- CPP/7zip/Bundles/Alone/Alone.dsp | 72 +- CPP/7zip/Bundles/Alone/makefile | 5 +- CPP/7zip/Bundles/Alone/resource.rc | 4 + CPP/7zip/Bundles/Alone7z/Alone.dsp | 72 +- CPP/7zip/Bundles/Alone7z/makefile | 6 +- CPP/7zip/Bundles/Alone7z/resource.rc | 4 + CPP/7zip/Bundles/Fm/FM.dsp | 34 +- CPP/7zip/Bundles/Fm/makefile | 1 - CPP/7zip/Bundles/Format7z/makefile | 3 + CPP/7zip/Bundles/Format7zExtract/makefile | 4 + CPP/7zip/Bundles/Format7zExtractR/makefile | 3 + CPP/7zip/Bundles/Format7zF/Arc.mak | 4 + CPP/7zip/Bundles/Format7zF/Format7z.dsp | 58 + CPP/7zip/Bundles/Format7zR/makefile | 3 + CPP/7zip/Bundles/SFXCon/SFXCon.dsp | 34 + CPP/7zip/Bundles/SFXCon/makefile | 5 + CPP/7zip/Bundles/SFXSetup/SFXSetup.dsp | 34 + CPP/7zip/Bundles/SFXSetup/makefile | 9 +- CPP/7zip/Bundles/SFXWin/SFXWin.dsp | 26 + CPP/7zip/Bundles/SFXWin/makefile | 6 + CPP/7zip/Common/CWrappers.cpp | 17 +- CPP/7zip/Common/CreateCoder.cpp | 113 +- CPP/7zip/Common/CreateCoder.h | 26 +- CPP/7zip/Common/InBuffer.cpp | 28 + CPP/7zip/Common/MethodProps.cpp | 13 +- CPP/7zip/Compress/BZip2Const.h | 11 + CPP/7zip/Compress/BZip2Decoder.cpp | 6 +- CPP/7zip/Compress/DeflateEncoder.cpp | 4 + CPP/7zip/Compress/LzOutWindow.h | 33 + CPP/7zip/Compress/LzfseDecoder.cpp | 925 +++++++ CPP/7zip/Compress/LzfseDecoder.h | 58 + CPP/7zip/Compress/Lzma2Decoder.cpp | 405 ++- CPP/7zip/Compress/Lzma2Decoder.h | 52 +- CPP/7zip/Compress/Lzma2Encoder.cpp | 2 +- CPP/7zip/Compress/LzmaDecoder.cpp | 11 +- CPP/7zip/Compress/LzmaDecoder.h | 6 +- CPP/7zip/Compress/LzmaEncoder.cpp | 6 +- CPP/7zip/Compress/Rar1Decoder.cpp | 403 +-- CPP/7zip/Compress/Rar1Decoder.h | 49 +- CPP/7zip/Compress/Rar2Decoder.cpp | 54 +- CPP/7zip/Compress/Rar2Decoder.h | 11 +- CPP/7zip/Compress/Rar3Decoder.cpp | 87 +- CPP/7zip/Compress/Rar3Decoder.h | 12 +- CPP/7zip/Compress/Rar5Decoder.cpp | 123 +- CPP/7zip/Compress/Rar5Decoder.h | 14 +- CPP/7zip/Compress/ShrinkDecoder.cpp | 77 +- CPP/7zip/Compress/ShrinkDecoder.h | 2 +- CPP/7zip/Compress/XzDecoder.cpp | 305 +-- CPP/7zip/Compress/XzDecoder.h | 97 +- CPP/7zip/Crypto/Rar5Aes.cpp | 10 +- CPP/7zip/GuiCommon.rc | 3 + CPP/7zip/Guid.txt | 4 +- CPP/7zip/ICoder.h | 10 +- CPP/7zip/LzmaDec.mak | 5 + CPP/7zip/UI/Agent/Agent.cpp | 26 +- CPP/7zip/UI/Agent/Agent.h | 6 + CPP/7zip/UI/Common/ArchiveCommandLine.cpp | 204 +- CPP/7zip/UI/Common/ArchiveCommandLine.h | 20 +- CPP/7zip/UI/Common/ArchiveExtractCallback.cpp | 179 +- CPP/7zip/UI/Common/ArchiveExtractCallback.h | 58 +- CPP/7zip/UI/Common/Bench.cpp | 65 +- CPP/7zip/UI/Common/CompressCall.cpp | 4 + CPP/7zip/UI/Common/CompressCall2.cpp | 5 +- CPP/7zip/UI/Common/EnumDirItems.cpp | 180 +- CPP/7zip/UI/Common/EnumDirItems.h | 20 + CPP/7zip/UI/Common/Extract.cpp | 10 +- CPP/7zip/UI/Common/HashCalc.h | 1 - CPP/7zip/UI/Common/OpenArchive.cpp | 8 +- CPP/7zip/UI/Common/OpenArchive.h | 19 + CPP/7zip/UI/Common/PropIDUtils.cpp | 132 +- CPP/7zip/UI/Common/Update.cpp | 66 +- CPP/7zip/UI/Common/Update.h | 2 + CPP/7zip/UI/Common/UpdateCallback.cpp | 11 +- CPP/7zip/UI/Common/UpdateCallback.h | 1 + CPP/7zip/UI/Console/Console.manifest | 13 + .../UI/Console/ExtractCallbackConsole.cpp | 60 +- CPP/7zip/UI/Console/HashCon.cpp | 2 +- CPP/7zip/UI/Console/List.cpp | 99 +- CPP/7zip/UI/Console/Main.cpp | 36 +- CPP/7zip/UI/Console/MainAr.cpp | 2 +- CPP/7zip/UI/Console/PercentPrinter.cpp | 8 +- CPP/7zip/UI/Console/UpdateCallbackConsole.cpp | 20 +- CPP/7zip/UI/Console/resource.rc | 4 + CPP/7zip/UI/Explorer/Explorer.dsp | 8 + CPP/7zip/UI/Explorer/makefile | 1 + CPP/7zip/UI/Far/ExtractEngine.cpp | 2 + CPP/7zip/UI/FileManager/EditDialog.cpp | 57 + CPP/7zip/UI/FileManager/EditDialog.h | 25 + CPP/7zip/UI/FileManager/EditDialog.rc | 15 + CPP/7zip/UI/FileManager/EditDialogRes.h | 2 + CPP/7zip/UI/FileManager/FM.cpp | 1 + CPP/7zip/UI/FileManager/FM.dsp | 8 + CPP/7zip/UI/FileManager/FM.mak | 1 + CPP/7zip/UI/FileManager/LangUtils.cpp | 1 + CPP/7zip/UI/FileManager/LinkDialog.cpp | 6 +- CPP/7zip/UI/FileManager/ListViewDialog.cpp | 210 +- CPP/7zip/UI/FileManager/ListViewDialog.h | 18 +- CPP/7zip/UI/FileManager/ListViewDialog.rc | 4 +- CPP/7zip/UI/FileManager/PanelCopy.cpp | 69 +- CPP/7zip/UI/FileManager/PanelCrc.cpp | 52 +- CPP/7zip/UI/FileManager/PanelDrag.cpp | 118 +- CPP/7zip/UI/FileManager/PanelFolderChange.cpp | 4 +- CPP/7zip/UI/FileManager/PanelItemOpen.cpp | 4 +- CPP/7zip/UI/FileManager/PanelListNotify.cpp | 2 + CPP/7zip/UI/FileManager/PanelMenu.cpp | 191 +- CPP/7zip/UI/FileManager/PanelOperations.cpp | 14 +- CPP/7zip/UI/FileManager/PanelSelect.cpp | 3 + CPP/7zip/UI/FileManager/PanelSplitFile.cpp | 121 +- CPP/7zip/UI/FileManager/ProgressDialog2.cpp | 30 +- CPP/7zip/UI/FileManager/ProgressDialog2.h | 70 +- CPP/7zip/UI/FileManager/resource.rc | 1 + CPP/7zip/UI/GUI/CompressDialog.cpp | 1 + CPP/7zip/UI/GUI/ExtractGUI.cpp | 81 +- CPP/7zip/UI/GUI/GUI.cpp | 27 +- CPP/7zip/UI/GUI/GUI.dsp | 32 + CPP/7zip/UI/GUI/HashGUI.cpp | 187 +- CPP/7zip/UI/GUI/HashGUI.h | 11 + CPP/7zip/UI/GUI/UpdateGUI.cpp | 6 +- CPP/7zip/UI/GUI/makefile | 4 + CPP/7zip/UI/GUI/resource.rc | 2 + CPP/Build.mak | 25 +- CPP/Common/MyString.h | 4 +- CPP/Common/MyWindows.h | 5 + CPP/Common/StdOutStream.cpp | 59 + CPP/Common/StdOutStream.h | 11 +- CPP/Windows/Control/Dialog.cpp | 1 + CPP/Windows/Control/Dialog.h | 3 + CPP/Windows/FileDir.cpp | 15 + CPP/Windows/FileIO.h | 13 +- CPP/Windows/FileLink.cpp | 8 +- CPP/Windows/MemoryLock.cpp | 25 + CPP/Windows/MemoryLock.h | 2 + CPP/Windows/Registry.cpp | 2 +- DOC/7zip.inf | 4 +- DOC/7zip.nsi | 6 +- DOC/7zip.wxs | 7 +- DOC/License.txt | 54 +- DOC/lzma.txt | 8 +- DOC/readme.txt | 6 +- DOC/src-history.txt | 21 + README.md | 16 +- 208 files changed, 13958 insertions(+), 3588 deletions(-) create mode 100644 Asm/x86/LzmaDecOpt.asm create mode 100644 C/Lzma2DecMt.c create mode 100644 C/Lzma2DecMt.h create mode 100644 C/MtDec.c create mode 100644 C/MtDec.h create mode 100644 CPP/7zip/Compress/LzfseDecoder.cpp create mode 100644 CPP/7zip/Compress/LzfseDecoder.h create mode 100644 CPP/7zip/LzmaDec.mak create mode 100644 CPP/7zip/UI/Console/Console.manifest create mode 100644 CPP/7zip/UI/FileManager/EditDialog.cpp create mode 100644 CPP/7zip/UI/FileManager/EditDialog.h create mode 100644 CPP/7zip/UI/FileManager/EditDialog.rc create mode 100644 CPP/7zip/UI/FileManager/EditDialogRes.h diff --git a/Asm/x86/7zAsm.asm b/Asm/x86/7zAsm.asm index 53a9e3ff..8c30d7b7 100644 --- a/Asm/x86/7zAsm.asm +++ b/Asm/x86/7zAsm.asm @@ -1,5 +1,5 @@ ; 7zAsm.asm -- ASM macros -; 2012-12-30 : Igor Pavlov : Public domain +; 2018-02-03 : Igor Pavlov : Public domain MY_ASM_START macro ifdef x64 @@ -52,6 +52,15 @@ endif x6 equ ESI x7 equ EDI + x0_W equ AX + x1_W equ CX + x2_W equ DX + x3_W equ BX + + x5_W equ BP + x6_W equ SI + x7_W equ DI + x0_L equ AL x1_L equ CL x2_L equ DL @@ -63,6 +72,10 @@ endif x3_H equ BH ifdef x64 + x5_L equ BPL + x6_L equ SIL + x7_L equ DIL + r0 equ RAX r1 equ RCX r2 equ RDX @@ -103,3 +116,32 @@ MY_POP_4_REGS macro pop r5 pop r3 endm + + +ifdef x64 + +; for WIN64-x64 ABI: + +REG_PARAM_0 equ r1 +REG_PARAM_1 equ r2 +REG_PARAM_2 equ r8 +REG_PARAM_3 equ r9 + +MY_PUSH_PRESERVED_REGS macro + MY_PUSH_4_REGS + push r12 + push r13 + push r14 + push r15 +endm + + +MY_POP_PRESERVED_REGS macro + pop r15 + pop r14 + pop r13 + pop r12 + MY_POP_4_REGS +endm + +endif diff --git a/Asm/x86/LzmaDecOpt.asm b/Asm/x86/LzmaDecOpt.asm new file mode 100644 index 00000000..0a89eb73 --- /dev/null +++ b/Asm/x86/LzmaDecOpt.asm @@ -0,0 +1,1258 @@ +; LzmaDecOpt.asm -- ASM version of LzmaDec_DecodeReal_3() function +; 2018-02-06: Igor Pavlov : Public domain +; +; 3 - is the code compatibility version of LzmaDec_DecodeReal_*() +; function for check at link time. +; That code is tightly coupled with LzmaDec_TryDummy() +; and with another functions in LzmaDec.c file. +; CLzmaDec structure, (probs) array layout, input and output of +; LzmaDec_DecodeReal_*() must be equal in both versions (C / ASM). + +ifndef x64 +; x64=1 +; .err +endif + +include 7zAsm.asm + +MY_ASM_START + +_TEXT$LZMADECOPT SEGMENT ALIGN(64) 'CODE' + +MY_ALIGN macro num:req + align num +endm + +MY_ALIGN_16 macro + MY_ALIGN 16 +endm + +MY_ALIGN_32 macro + MY_ALIGN 32 +endm + +MY_ALIGN_64 macro + MY_ALIGN 64 +endm + + +; _LZMA_SIZE_OPT equ 1 + +; _LZMA_PROB32 equ 1 + +ifdef _LZMA_PROB32 + PSHIFT equ 2 + PLOAD macro dest, mem + mov dest, dword ptr [mem] + endm + PSTORE macro src, mem + mov dword ptr [mem], src + endm +else + PSHIFT equ 1 + PLOAD macro dest, mem + movzx dest, word ptr [mem] + endm + PSTORE macro src, mem + mov word ptr [mem], @CatStr(src, _W) + endm +endif + +PMULT equ (1 SHL PSHIFT) +PMULT_HALF equ (1 SHL (PSHIFT - 1)) +PMULT_2 equ (1 SHL (PSHIFT + 1)) + + +; x0 range +; x1 pbPos / (prob) TREE +; x2 probBranch / prm (MATCHED) / pbPos / cnt +; x3 sym +;====== r4 === RSP +; x5 cod +; x6 t1 NORM_CALC / probs_state / dist +; x7 t0 NORM_CALC / prob2 IF_BIT_1 +; x8 state +; x9 match (MATCHED) / sym2 / dist2 / lpMask_reg +; x10 kBitModelTotal_reg +; r11 probs +; x12 offs (MATCHED) / dic / len_temp +; x13 processedPos +; x14 bit (MATCHED) / dicPos +; r15 buf + + +cod equ x5 +cod_L equ x5_L +range equ x0 +state equ x8 +state_R equ r8 +buf equ r15 +processedPos equ x13 +kBitModelTotal_reg equ x10 + +probBranch equ x2 +probBranch_R equ r2 +probBranch_W equ x2_W + +pbPos equ x1 +pbPos_R equ r1 + +cnt equ x2 +cnt_R equ r2 + +lpMask_reg equ x9 +dicPos equ r14 + +sym equ x3 +sym_R equ r3 +sym_L equ x3_L + +probs equ r11 +dic equ r12 + +t0 equ x7 +t0_W equ x7_W +t0_R equ r7 + +prob2 equ t0 +prob2_W equ t0_W + +t1 equ x6 +t1_R equ r6 + +probs_state equ t1 +probs_state_R equ t1_R + +prm equ r2 +match equ x9 +match_R equ r9 +offs equ x12 +offs_R equ r12 +bit equ x14 +bit_R equ r14 + +sym2 equ x9 +sym2_R equ r9 + +len_temp equ x12 + +dist equ sym +dist2 equ x9 + + + +kNumBitModelTotalBits equ 11 +kBitModelTotal equ (1 SHL kNumBitModelTotalBits) +kNumMoveBits equ 5 +kBitModelOffset equ ((1 SHL kNumMoveBits) - 1) +kTopValue equ (1 SHL 24) + +NORM_2 macro + ; movzx t0, BYTE PTR [buf] + shl cod, 8 + mov cod_L, BYTE PTR [buf] + shl range, 8 + ; or cod, t0 + inc buf +endm + + +NORM macro + cmp range, kTopValue + jae SHORT @F + NORM_2 +@@: +endm + + +; ---------- Branch MACROS ---------- + +UPDATE_0 macro probsArray:req, probOffset:req, probDisp:req + mov prob2, kBitModelTotal_reg + sub prob2, probBranch + shr prob2, kNumMoveBits + add probBranch, prob2 + PSTORE probBranch, probOffset * 1 + probsArray + probDisp * PMULT +endm + + +UPDATE_1 macro probsArray:req, probOffset:req, probDisp:req + sub prob2, range + sub cod, range + mov range, prob2 + mov prob2, probBranch + shr probBranch, kNumMoveBits + sub prob2, probBranch + PSTORE prob2, probOffset * 1 + probsArray + probDisp * PMULT +endm + + +CMP_COD macro probsArray:req, probOffset:req, probDisp:req + PLOAD probBranch, probOffset * 1 + probsArray + probDisp * PMULT + NORM + mov prob2, range + shr range, kNumBitModelTotalBits + imul range, probBranch + cmp cod, range +endm + + +IF_BIT_1_NOUP macro probsArray:req, probOffset:req, probDisp:req, toLabel:req + CMP_COD probsArray, probOffset, probDisp + jae toLabel +endm + + +IF_BIT_1 macro probsArray:req, probOffset:req, probDisp:req, toLabel:req + IF_BIT_1_NOUP probsArray, probOffset, probDisp, toLabel + UPDATE_0 probsArray, probOffset, probDisp +endm + + +IF_BIT_0_NOUP macro probsArray:req, probOffset:req, probDisp:req, toLabel:req + CMP_COD probsArray, probOffset, probDisp + jb toLabel +endm + + +; ---------- CMOV MACROS ---------- + +NORM_CALC macro prob:req + NORM + mov t0, range + shr range, kNumBitModelTotalBits + imul range, prob + sub t0, range + mov t1, cod + sub cod, range +endm + + +PUP macro prob:req, probPtr:req + sub t0, prob + ; only sar works for both 16/32 bit prob modes + sar t0, kNumMoveBits + add t0, prob + PSTORE t0, probPtr +endm + + +PUP_SUB macro prob:req, probPtr:req, symSub:req + sbb sym, symSub + PUP prob, probPtr +endm + + +PUP_COD macro prob:req, probPtr:req, symSub:req + mov t0, kBitModelOffset + cmovb cod, t1 + mov t1, sym + cmovb t0, kBitModelTotal_reg + PUP_SUB prob, probPtr, symSub +endm + + +BIT_0 macro prob:req, probNext:req + PLOAD prob, probs + 1 * PMULT + PLOAD probNext, probs + 1 * PMULT_2 + + NORM_CALC prob + + cmovae range, t0 + PLOAD t0, probs + 1 * PMULT_2 + PMULT + cmovae probNext, t0 + mov t0, kBitModelOffset + cmovb cod, t1 + cmovb t0, kBitModelTotal_reg + mov sym, 2 + PUP_SUB prob, probs + 1 * PMULT, 0 - 1 +endm + + +BIT_1 macro prob:req, probNext:req + PLOAD probNext, probs + sym_R * PMULT_2 + add sym, sym + + NORM_CALC prob + + cmovae range, t0 + PLOAD t0, probs + sym_R * PMULT + PMULT + cmovae probNext, t0 + PUP_COD prob, probs + t1_R * PMULT_HALF, 0 - 1 +endm + + +BIT_2 macro prob:req, symSub:req + add sym, sym + + NORM_CALC prob + + cmovae range, t0 + PUP_COD prob, probs + t1_R * PMULT_HALF, symSub +endm + + +; ---------- MATCHED LITERAL ---------- + +LITM_0 macro + mov offs, 256 * PMULT + shl match, (PSHIFT + 1) + mov bit, offs + and bit, match + PLOAD x1, probs + 256 * PMULT + bit_R * 1 + 1 * PMULT + lea prm, [probs + 256 * PMULT + bit_R * 1 + 1 * PMULT] + ; lea prm, [probs + 256 * PMULT + 1 * PMULT] + ; add prm, bit_R + xor offs, bit + add match, match + + NORM_CALC x1 + + cmovae offs, bit + mov bit, match + cmovae range, t0 + mov t0, kBitModelOffset + cmovb cod, t1 + cmovb t0, kBitModelTotal_reg + mov sym, 0 + PUP_SUB x1, prm, -2-1 +endm + + +LITM macro + and bit, offs + lea prm, [probs + offs_R * 1] + add prm, bit_R + PLOAD x1, prm + sym_R * PMULT + xor offs, bit + add sym, sym + add match, match + + NORM_CALC x1 + + cmovae offs, bit + mov bit, match + cmovae range, t0 + PUP_COD x1, prm + t1_R * PMULT_HALF, - 1 +endm + + +LITM_2 macro + and bit, offs + lea prm, [probs + offs_R * 1] + add prm, bit_R + PLOAD x1, prm + sym_R * PMULT + add sym, sym + + NORM_CALC x1 + + cmovae range, t0 + PUP_COD x1, prm + t1_R * PMULT_HALF, 256 - 1 +endm + + +; ---------- REVERSE BITS ---------- + +REV_0 macro prob:req, probNext:req + ; PLOAD prob, probs + 1 * PMULT + ; lea sym2_R, [probs + 2 * PMULT] + ; PLOAD probNext, probs + 2 * PMULT + PLOAD probNext, sym2_R + + NORM_CALC prob + + cmovae range, t0 + PLOAD t0, probs + 3 * PMULT + cmovae probNext, t0 + cmovb cod, t1 + mov t0, kBitModelOffset + cmovb t0, kBitModelTotal_reg + lea t1_R, [probs + 3 * PMULT] + cmovae sym2_R, t1_R + PUP prob, probs + 1 * PMULT +endm + + +REV_1 macro prob:req, probNext:req, step:req + add sym2_R, step * PMULT + PLOAD probNext, sym2_R + + NORM_CALC prob + + cmovae range, t0 + PLOAD t0, sym2_R + step * PMULT + cmovae probNext, t0 + cmovb cod, t1 + mov t0, kBitModelOffset + cmovb t0, kBitModelTotal_reg + lea t1_R, [sym2_R + step * PMULT] + cmovae sym2_R, t1_R + PUP prob, t1_R - step * PMULT_2 +endm + + +REV_2 macro prob:req, step:req + sub sym2_R, probs + shr sym2, PSHIFT + or sym, sym2 + + NORM_CALC prob + + cmovae range, t0 + lea t0, [sym - step] + cmovb sym, t0 + cmovb cod, t1 + mov t0, kBitModelOffset + cmovb t0, kBitModelTotal_reg + PUP prob, probs + sym2_R * PMULT +endm + + +REV_1_VAR macro prob:req + PLOAD prob, sym_R + mov probs, sym_R + add sym_R, sym2_R + + NORM_CALC prob + + cmovae range, t0 + lea t0_R, [sym_R + sym2_R] + cmovae sym_R, t0_R + mov t0, kBitModelOffset + cmovb cod, t1 + ; mov t1, kBitModelTotal + ; cmovb t0, t1 + cmovb t0, kBitModelTotal_reg + add sym2, sym2 + PUP prob, probs +endm + + + + +LIT_PROBS macro lpMaskParam:req + ; prob += (UInt32)3 * ((((processedPos << 8) + dic[(dicPos == 0 ? dicBufSize : dicPos) - 1]) & lpMask) << lc); + mov t0, processedPos + shl t0, 8 + add sym, t0 + and sym, lpMaskParam + add probs_state_R, pbPos_R + mov x1, LOC lc2 + lea sym, dword ptr[sym_R + 2 * sym_R] + add probs, Literal * PMULT + shl sym, x1_L + add probs, sym_R + UPDATE_0 probs_state_R, 0, IsMatch + inc processedPos +endm + + + +kNumPosBitsMax equ 4 +kNumPosStatesMax equ (1 SHL kNumPosBitsMax) + +kLenNumLowBits equ 3 +kLenNumLowSymbols equ (1 SHL kLenNumLowBits) +kLenNumHighBits equ 8 +kLenNumHighSymbols equ (1 SHL kLenNumHighBits) +kNumLenProbs equ (2 * kLenNumLowSymbols * kNumPosStatesMax + kLenNumHighSymbols) + +LenLow equ 0 +LenChoice equ LenLow +LenChoice2 equ (LenLow + kLenNumLowSymbols) +LenHigh equ (LenLow + 2 * kLenNumLowSymbols * kNumPosStatesMax) + +kNumStates equ 12 +kNumStates2 equ 16 +kNumLitStates equ 7 + +kStartPosModelIndex equ 4 +kEndPosModelIndex equ 14 +kNumFullDistances equ (1 SHL (kEndPosModelIndex SHR 1)) + +kNumPosSlotBits equ 6 +kNumLenToPosStates equ 4 + +kNumAlignBits equ 4 +kAlignTableSize equ (1 SHL kNumAlignBits) + +kMatchMinLen equ 2 +kMatchSpecLenStart equ (kMatchMinLen + kLenNumLowSymbols * 2 + kLenNumHighSymbols) + +kStartOffset equ 1664 +SpecPos equ (-kStartOffset) +IsRep0Long equ (SpecPos + kNumFullDistances) +RepLenCoder equ (IsRep0Long + (kNumStates2 SHL kNumPosBitsMax)) +LenCoder equ (RepLenCoder + kNumLenProbs) +IsMatch equ (LenCoder + kNumLenProbs) +kAlign equ (IsMatch + (kNumStates2 SHL kNumPosBitsMax)) +IsRep equ (kAlign + kAlignTableSize) +IsRepG0 equ (IsRep + kNumStates) +IsRepG1 equ (IsRepG0 + kNumStates) +IsRepG2 equ (IsRepG1 + kNumStates) +PosSlot equ (IsRepG2 + kNumStates) +Literal equ (PosSlot + (kNumLenToPosStates SHL kNumPosSlotBits)) +NUM_BASE_PROBS equ (Literal + kStartOffset) + +if kAlign ne 0 + .err +endif + +if NUM_BASE_PROBS ne 1984 + .err +endif + + +PTR_FIELD equ dq ? + +CLzmaDec_Asm struct + lc db ? + lp db ? + pb db ? + _pad_ db ? + dicSize dd ? + + probs_Spec PTR_FIELD + probs_1664 PTR_FIELD + dic_Spec PTR_FIELD + dicBufSize PTR_FIELD + dicPos_Spec PTR_FIELD + buf_Spec PTR_FIELD + + range_Spec dd ? + code_Spec dd ? + processedPos_Spec dd ? + checkDicSize dd ? + rep0 dd ? + rep1 dd ? + rep2 dd ? + rep3 dd ? + state_Spec dd ? + remainLen dd ? +CLzmaDec_Asm ends + + +CLzmaDec_Asm_Loc struct + OLD_RSP PTR_FIELD + lzmaPtr PTR_FIELD + _pad0_ PTR_FIELD + _pad1_ PTR_FIELD + _pad2_ PTR_FIELD + dicBufSize PTR_FIELD + probs_Spec PTR_FIELD + dic_Spec PTR_FIELD + + limit PTR_FIELD + bufLimit PTR_FIELD + lc2 dd ? + lpMask dd ? + pbMask dd ? + checkDicSize dd ? + + _pad_ dd ? + remainLen dd ? + dicPos_Spec PTR_FIELD + rep0 dd ? + rep1 dd ? + rep2 dd ? + rep3 dd ? +CLzmaDec_Asm_Loc ends + + +GLOB_2 equ [sym_R].CLzmaDec_Asm. +GLOB equ [r1].CLzmaDec_Asm. +LOC_0 equ [r0].CLzmaDec_Asm_Loc. +LOC equ [RSP].CLzmaDec_Asm_Loc. + + +COPY_VAR macro name + mov t0, GLOB_2 name + mov LOC_0 name, t0 +endm + + +RESTORE_VAR macro name + mov t0, LOC name + mov GLOB name, t0 +endm + + + +IsMatchBranch_Pre macro reg + ; prob = probs + IsMatch + (state << kNumPosBitsMax) + posState; + mov pbPos, LOC pbMask + and pbPos, processedPos + shl pbPos, (kLenNumLowBits + 1 + PSHIFT) + lea probs_state_R, [probs + state_R] +endm + + +IsMatchBranch macro reg + IsMatchBranch_Pre + IF_BIT_1 probs_state_R, pbPos_R, IsMatch, IsMatch_label +endm + + +CheckLimits macro reg + cmp buf, LOC bufLimit + jae fin_OK + cmp dicPos, LOC limit + jae fin_OK +endm + + + +; RSP is (16x + 8) bytes aligned in WIN64-x64 +; LocalSize equ ((((SIZEOF CLzmaDec_Asm_Loc) + 7) / 16 * 16) + 8) + +PARAM_lzma equ REG_PARAM_0 +PARAM_limit equ REG_PARAM_1 +PARAM_bufLimit equ REG_PARAM_2 + +; MY_ALIGN_64 +MY_PROC LzmaDec_DecodeReal_3, 3 +MY_PUSH_PRESERVED_REGS + + lea r0, [RSP - (SIZEOF CLzmaDec_Asm_Loc)] + and r0, -128 + mov r5, RSP + mov RSP, r0 + mov LOC_0 Old_RSP, r5 + mov LOC_0 lzmaPtr, PARAM_lzma + + mov LOC_0 remainLen, 0 ; remainLen must be ZERO + + mov LOC_0 bufLimit, PARAM_bufLimit + mov sym_R, PARAM_lzma ; CLzmaDec_Asm_Loc pointer for GLOB_2 + mov dic, GLOB_2 dic_Spec + add PARAM_limit, dic + mov LOC_0 limit, PARAM_limit + + COPY_VAR(rep0) + COPY_VAR(rep1) + COPY_VAR(rep2) + COPY_VAR(rep3) + + mov dicPos, GLOB_2 dicPos_Spec + add dicPos, dic + mov LOC_0 dicPos_Spec, dicPos + mov LOC_0 dic_Spec, dic + + mov x1_L, GLOB_2 pb + mov t0, 1 + shl t0, x1_L + dec t0 + mov LOC_0 pbMask, t0 + + ; unsigned pbMask = ((unsigned)1 << (p->prop.pb)) - 1; + ; unsigned lc = p->prop.lc; + ; unsigned lpMask = ((unsigned)0x100 << p->prop.lp) - ((unsigned)0x100 >> lc); + + mov x1_L, GLOB_2 lc + mov x2, 100h + mov t0, x2 + shr x2, x1_L + ; inc x1 + add x1_L, PSHIFT + mov LOC_0 lc2, x1 + mov x1_L, GLOB_2 lp + shl t0, x1_L + sub t0, x2 + mov LOC_0 lpMask, t0 + mov lpMask_reg, t0 + + ; mov probs, GLOB_2 probs_Spec + ; add probs, kStartOffset SHL PSHIFT + mov probs, GLOB_2 probs_1664 + mov LOC_0 probs_Spec, probs + + mov t0_R, GLOB_2 dicBufSize + mov LOC_0 dicBufSize, t0_R + + mov x1, GLOB_2 checkDicSize + mov LOC_0 checkDicSize, x1 + + mov processedPos, GLOB_2 processedPos_Spec + + mov state, GLOB_2 state_Spec + shl state, PSHIFT + + mov buf, GLOB_2 buf_Spec + mov range, GLOB_2 range_Spec + mov cod, GLOB_2 code_Spec + mov kBitModelTotal_reg, kBitModelTotal + xor sym, sym + + ; if (processedPos != 0 || checkDicSize != 0) + or x1, processedPos + jz @f + + add t0_R, dic + cmp dicPos, dic + cmovnz t0_R, dicPos + movzx sym, byte ptr[t0_R - 1] + +@@: + IsMatchBranch_Pre + cmp state, 4 * PMULT + jb lit_end + cmp state, kNumLitStates * PMULT + jb lit_matched_end + jmp lz_end + + + + +; ---------- LITERAL ---------- +MY_ALIGN_64 +lit_start: + xor state, state +lit_start_2: + LIT_PROBS lpMask_reg + + ifdef _LZMA_SIZE_OPT + + PLOAD x1, probs + 1 * PMULT + mov sym, 1 +MY_ALIGN_16 +lit_loop: + BIT_1 x1, x2 + mov x1, x2 + cmp sym, 127 + jbe lit_loop + + else + + BIT_0 x1, x2 + BIT_1 x2, x1 + BIT_1 x1, x2 + BIT_1 x2, x1 + BIT_1 x1, x2 + BIT_1 x2, x1 + BIT_1 x1, x2 + + endif + + BIT_2 x2, 256 - 1 + + ; mov dic, LOC dic_Spec + mov probs, LOC probs_Spec + IsMatchBranch_Pre + mov byte ptr[dicPos], sym_L + inc dicPos + + CheckLimits +lit_end: + IF_BIT_0_NOUP probs_state_R, pbPos_R, IsMatch, lit_start + + ; jmp IsMatch_label + +; ---------- MATCHES ---------- +; MY_ALIGN_32 +IsMatch_label: + UPDATE_1 probs_state_R, pbPos_R, IsMatch + IF_BIT_1 probs_state_R, 0, IsRep, IsRep_label + + add probs, LenCoder * PMULT + add state, kNumStates * PMULT + +; ---------- LEN DECODE ---------- +len_decode: + mov len_temp, 8 - 1 - kMatchMinLen + IF_BIT_0_NOUP probs, 0, 0, len_mid_0 + UPDATE_1 probs, 0, 0 + add probs, (1 SHL (kLenNumLowBits + PSHIFT)) + mov len_temp, -1 - kMatchMinLen + IF_BIT_0_NOUP probs, 0, 0, len_mid_0 + UPDATE_1 probs, 0, 0 + add probs, LenHigh * PMULT - (1 SHL (kLenNumLowBits + PSHIFT)) + mov sym, 1 + PLOAD x1, probs + 1 * PMULT + +MY_ALIGN_32 +len8_loop: + BIT_1 x1, x2 + mov x1, x2 + cmp sym, 64 + jb len8_loop + + mov len_temp, (kLenNumHighSymbols - kLenNumLowSymbols * 2) - 1 - kMatchMinLen + jmp len_mid_2 + +MY_ALIGN_32 +len_mid_0: + UPDATE_0 probs, 0, 0 + add probs, pbPos_R + BIT_0 x2, x1 +len_mid_2: + BIT_1 x1, x2 + BIT_2 x2, len_temp + mov probs, LOC probs_Spec + cmp state, kNumStates * PMULT + jb copy_match + + +; ---------- DECODE DISTANCE ---------- + ; probs + PosSlot + ((len < kNumLenToPosStates ? len : kNumLenToPosStates - 1) << kNumPosSlotBits); + + mov t0, 3 + kMatchMinLen + cmp sym, 3 + kMatchMinLen + cmovb t0, sym + add probs, PosSlot * PMULT - (kMatchMinLen SHL (kNumPosSlotBits + PSHIFT)) + shl t0, (kNumPosSlotBits + PSHIFT) + add probs, t0_R + + ; sym = Len + ; mov LOC remainLen, sym + mov len_temp, sym + + ifdef _LZMA_SIZE_OPT + + PLOAD x1, probs + 1 * PMULT + mov sym, 1 +MY_ALIGN_16 +slot_loop: + BIT_1 x1, x2 + mov x1, x2 + cmp sym, 32 + jb slot_loop + + else + + BIT_0 x1, x2 + BIT_1 x2, x1 + BIT_1 x1, x2 + BIT_1 x2, x1 + BIT_1 x1, x2 + + endif + + mov x1, sym + BIT_2 x2, 64-1 + + and sym, 3 + mov probs, LOC probs_Spec + cmp x1, 32 + kEndPosModelIndex / 2 + jb short_dist + + ; unsigned numDirectBits = (unsigned)(((distance >> 1) - 1)); + sub x1, (32 + 1 + kNumAlignBits) + ; distance = (2 | (distance & 1)); + or sym, 2 + PLOAD x2, probs + 1 * PMULT + shl sym, kNumAlignBits + 1 + lea sym2_R, [probs + 2 * PMULT] + + jmp direct_norm + ; lea t1, [sym_R + (1 SHL kNumAlignBits)] + ; cmp range, kTopValue + ; jb direct_norm + +; ---------- DIRECT DISTANCE ---------- +MY_ALIGN_32 +direct_loop: + shr range, 1 + mov t0, cod + sub cod, range + cmovs cod, t0 + cmovns sym, t1 + + comment ~ + sub cod, range + mov x2, cod + sar x2, 31 + lea sym, dword ptr [r2 + sym_R * 2 + 1] + and x2, range + add cod, x2 + ~ + dec x1 + je direct_end + + add sym, sym +direct_norm: + lea t1, [sym_R + (1 SHL kNumAlignBits)] + cmp range, kTopValue + jae near ptr direct_loop + ; we align for 32 here with "near ptr" command above + NORM_2 + jmp direct_loop + +MY_ALIGN_32 +direct_end: + ; prob = + kAlign; + ; distance <<= kNumAlignBits; + REV_0 x2, x1 + REV_1 x1, x2, 2 + REV_1 x2, x1, 4 + REV_2 x1, 8 + +decode_dist_end: + + ; if (distance >= (checkDicSize == 0 ? processedPos: checkDicSize)) + + mov t0, LOC checkDicSize + test t0, t0 + cmove t0, processedPos + cmp sym, t0 + jae end_of_payload + + ; rep3 = rep2; + ; rep2 = rep1; + ; rep1 = rep0; + ; rep0 = distance + 1; + + inc sym + mov t0, LOC rep0 + mov t1, LOC rep1 + mov x1, LOC rep2 + mov LOC rep0, sym + ; mov sym, LOC remainLen + mov sym, len_temp + mov LOC rep1, t0 + mov LOC rep2, t1 + mov LOC rep3, x1 + + ; state = (state < kNumStates + kNumLitStates) ? kNumLitStates : kNumLitStates + 3; + cmp state, (kNumStates + kNumLitStates) * PMULT + mov state, kNumLitStates * PMULT + mov t0, (kNumLitStates + 3) * PMULT + cmovae state, t0 + + +; ---------- COPY MATCH ---------- +copy_match: + + ; len += kMatchMinLen; + ; add sym, kMatchMinLen + + ; if ((rem = limit - dicPos) == 0) + ; { + ; p->dicPos = dicPos; + ; return SZ_ERROR_DATA; + ; } + mov cnt_R, LOC limit + sub cnt_R, dicPos + jz fin_ERROR + + ; curLen = ((rem < len) ? (unsigned)rem : len); + cmp cnt_R, sym_R + ; cmovae cnt_R, sym_R ; 64-bit + cmovae cnt, sym ; 32-bit + + mov dic, LOC dic_Spec + mov x1, LOC rep0 + + mov t0_R, dicPos + add dicPos, cnt_R + ; processedPos += curLen; + add processedPos, cnt + ; len -= curLen; + sub sym, cnt + mov LOC remainLen, sym + + sub t0_R, dic + + ; pos = dicPos - rep0 + (dicPos < rep0 ? dicBufSize : 0); + sub t0_R, r1 + jae @f + + mov r1, LOC dicBufSize + add t0_R, r1 + sub r1, t0_R + cmp cnt_R, r1 + ja copy_match_cross +@@: + ; if (curLen <= dicBufSize - pos) + +; ---------- COPY MATCH FAST ---------- + ; Byte *dest = dic + dicPos; + ; mov r1, dic + ; ptrdiff_t src = (ptrdiff_t)pos - (ptrdiff_t)dicPos; + ; sub t0_R, dicPos + ; dicPos += curLen; + + ; const Byte *lim = dest + curLen; + add t0_R, dic + movzx sym, byte ptr[t0_R] + add t0_R, cnt_R + neg cnt_R + ; lea r1, [dicPos - 1] +copy_common: + dec dicPos + ; cmp LOC rep0, 1 + ; je rep0Label + + ; t0_R - src_lim + ; r1 - dest_lim - 1 + ; cnt_R - (-cnt) + + IsMatchBranch_Pre + inc cnt_R + jz copy_end +MY_ALIGN_16 +@@: + mov byte ptr[cnt_R * 1 + dicPos], sym_L + movzx sym, byte ptr[cnt_R * 1 + t0_R] + inc cnt_R + jnz @b + +copy_end: +lz_end_match: + mov byte ptr[dicPos], sym_L + inc dicPos + + ; IsMatchBranch_Pre + CheckLimits +lz_end: + IF_BIT_1_NOUP probs_state_R, pbPos_R, IsMatch, IsMatch_label + + + +; ---------- LITERAL MATCHED ---------- + + LIT_PROBS LOC lpMask + + ; matchByte = dic[dicPos - rep0 + (dicPos < rep0 ? dicBufSize : 0)]; + mov x1, LOC rep0 + ; mov dic, LOC dic_Spec + mov LOC dicPos_Spec, dicPos + + ; state -= (state < 10) ? 3 : 6; + lea t0, [state_R - 6 * PMULT] + sub state, 3 * PMULT + cmp state, 7 * PMULT + cmovae state, t0 + + sub dicPos, dic + sub dicPos, r1 + jae @f + add dicPos, LOC dicBufSize +@@: + comment ~ + xor t0, t0 + sub dicPos, r1 + cmovb t0_R, LOC dicBufSize + ~ + + movzx match, byte ptr[dic + dicPos * 1] + + ifdef _LZMA_SIZE_OPT + + mov offs, 256 * PMULT + shl match, (PSHIFT + 1) + mov bit, match + mov sym, 1 +MY_ALIGN_16 +litm_loop: + LITM + cmp sym, 256 + jb litm_loop + sub sym, 256 + + else + + LITM_0 + LITM + LITM + LITM + LITM + LITM + LITM + LITM_2 + + endif + + mov probs, LOC probs_Spec + IsMatchBranch_Pre + ; mov dic, LOC dic_Spec + mov dicPos, LOC dicPos_Spec + mov byte ptr[dicPos], sym_L + inc dicPos + + CheckLimits +lit_matched_end: + IF_BIT_1_NOUP probs_state_R, pbPos_R, IsMatch, IsMatch_label + ; IsMatchBranch + mov lpMask_reg, LOC lpMask + sub state, 3 * PMULT + jmp lit_start_2 + + + +; ---------- REP 0 LITERAL ---------- +MY_ALIGN_32 +IsRep0Short_label: + UPDATE_0 probs_state_R, pbPos_R, IsRep0Long + + ; dic[dicPos] = dic[dicPos - rep0 + (dicPos < rep0 ? dicBufSize : 0)]; + mov dic, LOC dic_Spec + mov t0_R, dicPos + mov probBranch, LOC rep0 + sub t0_R, dic + + sub probs, RepLenCoder * PMULT + inc processedPos + ; state = state < kNumLitStates ? 9 : 11; + or state, 1 * PMULT + IsMatchBranch_Pre + + sub t0_R, probBranch_R + jae @f + add t0_R, LOC dicBufSize +@@: + movzx sym, byte ptr[dic + t0_R * 1] + jmp lz_end_match + + +MY_ALIGN_32 +IsRep_label: + UPDATE_1 probs_state_R, 0, IsRep + + ; The (checkDicSize == 0 && processedPos == 0) case was checked before in LzmaDec.c with kBadRepCode. + ; So we don't check it here. + + ; mov t0, processedPos + ; or t0, LOC checkDicSize + ; jz fin_ERROR_2 + + ; state = state < kNumLitStates ? 8 : 11; + cmp state, kNumLitStates * PMULT + mov state, 8 * PMULT + mov probBranch, 11 * PMULT + cmovae state, probBranch + + ; prob = probs + RepLenCoder; + add probs, RepLenCoder * PMULT + + IF_BIT_1 probs_state_R, 0, IsRepG0, IsRepG0_label + IF_BIT_0_NOUP probs_state_R, pbPos_R, IsRep0Long, IsRep0Short_label + UPDATE_1 probs_state_R, pbPos_R, IsRep0Long + jmp len_decode + +MY_ALIGN_32 +IsRepG0_label: + UPDATE_1 probs_state_R, 0, IsRepG0 + mov dist2, LOC rep0 + mov dist, LOC rep1 + mov LOC rep1, dist2 + + IF_BIT_1 probs_state_R, 0, IsRepG1, IsRepG1_label + mov LOC rep0, dist + jmp len_decode + +; MY_ALIGN_32 +IsRepG1_label: + UPDATE_1 probs_state_R, 0, IsRepG1 + mov dist2, LOC rep2 + mov LOC rep2, dist + + IF_BIT_1 probs_state_R, 0, IsRepG2, IsRepG2_label + mov LOC rep0, dist2 + jmp len_decode + +; MY_ALIGN_32 +IsRepG2_label: + UPDATE_1 probs_state_R, 0, IsRepG2 + mov dist, LOC rep3 + mov LOC rep3, dist2 + mov LOC rep0, dist + jmp len_decode + + + +; ---------- SPEC SHORT DISTANCE ---------- + +MY_ALIGN_32 +short_dist: + sub x1, 32 + 1 + jbe decode_dist_end + or sym, 2 + shl sym, x1_L + lea sym_R, [probs + sym_R * PMULT + SpecPos * PMULT + 1 * PMULT] + mov sym2, PMULT ; step +MY_ALIGN_32 +spec_loop: + REV_1_VAR x2 + dec x1 + jnz spec_loop + + mov probs, LOC probs_Spec + sub sym, sym2 + sub sym, SpecPos * PMULT + sub sym_R, probs + shr sym, PSHIFT + + jmp decode_dist_end + + +; ---------- COPY MATCH CROSS ---------- +copy_match_cross: + ; t0_R - src pos + ; r1 - len to dicBufSize + ; cnt_R - total copy len + + mov t1_R, t0_R ; srcPos + mov t0_R, dic + mov r1, LOC dicBufSize ; + neg cnt_R +@@: + movzx sym, byte ptr[t1_R * 1 + t0_R] + inc t1_R + mov byte ptr[cnt_R * 1 + dicPos], sym_L + inc cnt_R + cmp t1_R, r1 + jne @b + + movzx sym, byte ptr[t0_R] + sub t0_R, cnt_R + jmp copy_common + + + + +fin_ERROR: + mov LOC remainLen, len_temp +; fin_ERROR_2: + mov sym, 1 + jmp fin + +end_of_payload: + cmp sym, 0FFFFFFFFh ; -1 + jne fin_ERROR + + mov LOC remainLen, kMatchSpecLenStart + sub state, kNumStates * PMULT + +fin_OK: + xor sym, sym + +fin: + NORM + + mov r1, LOC lzmaPtr + + sub dicPos, LOC dic_Spec + mov GLOB dicPos_Spec, dicPos + mov GLOB buf_Spec, buf + mov GLOB range_Spec, range + mov GLOB code_Spec, cod + shr state, PSHIFT + mov GLOB state_Spec, state + mov GLOB processedPos_Spec, processedPos + + RESTORE_VAR(remainLen) + RESTORE_VAR(rep0) + RESTORE_VAR(rep1) + RESTORE_VAR(rep2) + RESTORE_VAR(rep3) + + mov x0, sym + + mov RSP, LOC Old_RSP + +MY_POP_PRESERVED_REGS +MY_ENDP + +_TEXT$LZMADECOPT ENDS + +end diff --git a/C/7zVersion.h b/C/7zVersion.h index 1c601aea..bedb8580 100644 --- a/C/7zVersion.h +++ b/C/7zVersion.h @@ -1,7 +1,7 @@ -#define MY_VER_MAJOR 17 -#define MY_VER_MINOR 01 +#define MY_VER_MAJOR 18 +#define MY_VER_MINOR 05 #define MY_VER_BUILD 0 -#define MY_VERSION_NUMBERS "17.01 ZS v1.3.2 R1" +#define MY_VERSION_NUMBERS "18.05 ZS v1.3.6 R1" #define MY_VERSION MY_VERSION_NUMBERS #ifdef MY_CPU_NAME @@ -10,12 +10,12 @@ #define MY_VERSION_CPU MY_VERSION #endif -#define MY_DATE "2017-10-31" +#define MY_DATE "2018-10-21" #undef MY_COPYRIGHT #undef MY_VERSION_COPYRIGHT_DATE #define MY_AUTHOR_NAME "Igor Pavlov, Tino Reichardt" #define MY_COPYRIGHT_PD "Igor Pavlov : Public domain" -#define MY_COPYRIGHT_CR "Copyright (c) 1999-2017 Igor Pavlov" +#define MY_COPYRIGHT_CR "Copyright (c) 1999-2018 Igor Pavlov" #ifdef USE_COPYRIGHT_CR #define MY_COPYRIGHT MY_COPYRIGHT_CR diff --git a/C/Alloc.c b/C/Alloc.c index 9f7cae3d..30b499e5 100644 --- a/C/Alloc.c +++ b/C/Alloc.c @@ -1,8 +1,10 @@ /* Alloc.c -- Memory allocation functions -2017-06-15 : Igor Pavlov : Public domain */ +2018-04-27 : Igor Pavlov : Public domain */ #include "Precomp.h" +#include + #ifdef _WIN32 #include #endif @@ -14,12 +16,119 @@ /* use _SZ_ALLOC_DEBUG to debug alloc/free operations */ #ifdef _SZ_ALLOC_DEBUG + #include int g_allocCount = 0; int g_allocCountMid = 0; int g_allocCountBig = 0; + + +#define CONVERT_INT_TO_STR(charType, tempSize) \ + unsigned char temp[tempSize]; unsigned i = 0; \ + while (val >= 10) { temp[i++] = (unsigned char)('0' + (unsigned)(val % 10)); val /= 10; } \ + *s++ = (charType)('0' + (unsigned)val); \ + while (i != 0) { i--; *s++ = temp[i]; } \ + *s = 0; + +static void ConvertUInt64ToString(UInt64 val, char *s) +{ + CONVERT_INT_TO_STR(char, 24); +} + +#define GET_HEX_CHAR(t) ((char)(((t < 10) ? ('0' + t) : ('A' + (t - 10))))) + +static void ConvertUInt64ToHex(UInt64 val, char *s) +{ + UInt64 v = val; + unsigned i; + for (i = 1;; i++) + { + v >>= 4; + if (v == 0) + break; + } + s[i] = 0; + do + { + unsigned t = (unsigned)(val & 0xF); + val >>= 4; + s[--i] = GET_HEX_CHAR(t); + } + while (i); +} + +#define DEBUG_OUT_STREAM stderr + +static void Print(const char *s) +{ + fputs(s, DEBUG_OUT_STREAM); +} + +static void PrintAligned(const char *s, size_t align) +{ + size_t len = strlen(s); + for(;;) + { + fputc(' ', DEBUG_OUT_STREAM); + if (len >= align) + break; + ++len; + } + Print(s); +} + +static void PrintLn() +{ + Print("\n"); +} + +static void PrintHex(UInt64 v, size_t align) +{ + char s[32]; + ConvertUInt64ToHex(v, s); + PrintAligned(s, align); +} + +static void PrintDec(UInt64 v, size_t align) +{ + char s[32]; + ConvertUInt64ToString(v, s); + PrintAligned(s, align); +} + +static void PrintAddr(void *p) +{ + PrintHex((UInt64)(size_t)(ptrdiff_t)p, 12); +} + + +#define PRINT_ALLOC(name, cnt, size, ptr) \ + Print(name " "); \ + PrintDec(cnt++, 10); \ + PrintHex(size, 10); \ + PrintAddr(ptr); \ + PrintLn(); + +#define PRINT_FREE(name, cnt, ptr) if (ptr) { \ + Print(name " "); \ + PrintDec(--cnt, 10); \ + PrintAddr(ptr); \ + PrintLn(); } + +#else + +#define PRINT_ALLOC(name, cnt, size, ptr) +#define PRINT_FREE(name, cnt, ptr) +#define Print(s) +#define PrintLn() +#define PrintHex(v, align) +#define PrintDec(v, align) +#define PrintAddr(p) + #endif + + void *MyAlloc(size_t size) { if (size == 0) @@ -27,7 +136,7 @@ void *MyAlloc(size_t size) #ifdef _SZ_ALLOC_DEBUG { void *p = malloc(size); - fprintf(stderr, "\nAlloc %10u bytes, count = %10d, addr = %8X", size, g_allocCount++, (unsigned)p); + PRINT_ALLOC("Alloc ", g_allocCount, size, p); return p; } #else @@ -37,10 +146,8 @@ void *MyAlloc(size_t size) void MyFree(void *address) { - #ifdef _SZ_ALLOC_DEBUG - if (address) - fprintf(stderr, "\nFree; count = %10d, addr = %8X", --g_allocCount, (unsigned)address); - #endif + PRINT_FREE("Free ", g_allocCount, address); + free(address); } @@ -50,18 +157,16 @@ void *MidAlloc(size_t size) { if (size == 0) return NULL; - #ifdef _SZ_ALLOC_DEBUG - fprintf(stderr, "\nAlloc_Mid %10d bytes; count = %10d", size, g_allocCountMid++); - #endif + + PRINT_ALLOC("Alloc-Mid", g_allocCountMid, size, NULL); + return VirtualAlloc(NULL, size, MEM_COMMIT, PAGE_READWRITE); } void MidFree(void *address) { - #ifdef _SZ_ALLOC_DEBUG - if (address) - fprintf(stderr, "\nFree_Mid; count = %10d", --g_allocCountMid); - #endif + PRINT_FREE("Free-Mid", g_allocCountMid, address); + if (!address) return; VirtualFree(address, 0, MEM_RELEASE); @@ -96,9 +201,8 @@ void *BigAlloc(size_t size) { if (size == 0) return NULL; - #ifdef _SZ_ALLOC_DEBUG - fprintf(stderr, "\nAlloc_Big %10u bytes; count = %10d", size, g_allocCountBig++); - #endif + + PRINT_ALLOC("Alloc-Big", g_allocCountBig, size, NULL); #ifdef _7ZIP_LARGE_PAGES { @@ -123,10 +227,7 @@ void *BigAlloc(size_t size) void BigFree(void *address) { - #ifdef _SZ_ALLOC_DEBUG - if (address) - fprintf(stderr, "\nFree_Big; count = %10d", --g_allocCountBig); - #endif + PRINT_FREE("Free-Big", g_allocCountBig, address); if (!address) return; @@ -138,8 +239,217 @@ void BigFree(void *address) static void *SzAlloc(ISzAllocPtr p, size_t size) { UNUSED_VAR(p); return MyAlloc(size); } static void SzFree(ISzAllocPtr p, void *address) { UNUSED_VAR(p); MyFree(address); } -ISzAlloc const g_Alloc = { SzAlloc, SzFree }; +const ISzAlloc g_Alloc = { SzAlloc, SzFree }; + +static void *SzMidAlloc(ISzAllocPtr p, size_t size) { UNUSED_VAR(p); return MidAlloc(size); } +static void SzMidFree(ISzAllocPtr p, void *address) { UNUSED_VAR(p); MidFree(address); } +const ISzAlloc g_MidAlloc = { SzMidAlloc, SzMidFree }; static void *SzBigAlloc(ISzAllocPtr p, size_t size) { UNUSED_VAR(p); return BigAlloc(size); } static void SzBigFree(ISzAllocPtr p, void *address) { UNUSED_VAR(p); BigFree(address); } -ISzAlloc const g_BigAlloc = { SzBigAlloc, SzBigFree }; +const ISzAlloc g_BigAlloc = { SzBigAlloc, SzBigFree }; + + +/* + uintptr_t : C99 (optional) + : unsupported in VS6 +*/ + +#ifdef _WIN32 + typedef UINT_PTR UIntPtr; +#else + /* + typedef uintptr_t UIntPtr; + */ + typedef ptrdiff_t UIntPtr; +#endif + + +#define ADJUST_ALLOC_SIZE 0 +/* +#define ADJUST_ALLOC_SIZE (sizeof(void *) - 1) +*/ +/* + Use (ADJUST_ALLOC_SIZE = (sizeof(void *) - 1)), if + MyAlloc() can return address that is NOT multiple of sizeof(void *). +*/ + + +/* +#define MY_ALIGN_PTR_DOWN(p, align) ((void *)((char *)(p) - ((size_t)(UIntPtr)(p) & ((align) - 1)))) +*/ +#define MY_ALIGN_PTR_DOWN(p, align) ((void *)((((UIntPtr)(p)) & ~((UIntPtr)(align) - 1)))) + +#define MY_ALIGN_PTR_UP_PLUS(p, align) MY_ALIGN_PTR_DOWN(((char *)(p) + (align) + ADJUST_ALLOC_SIZE), align) + + +#if (_POSIX_C_SOURCE >= 200112L) && !defined(_WIN32) + #define USE_posix_memalign +#endif + +/* + This posix_memalign() is for test purposes only. + We also need special Free() function instead of free(), + if this posix_memalign() is used. +*/ + +/* +static int posix_memalign(void **ptr, size_t align, size_t size) +{ + size_t newSize = size + align; + void *p; + void *pAligned; + *ptr = NULL; + if (newSize < size) + return 12; // ENOMEM + p = MyAlloc(newSize); + if (!p) + return 12; // ENOMEM + pAligned = MY_ALIGN_PTR_UP_PLUS(p, align); + ((void **)pAligned)[-1] = p; + *ptr = pAligned; + return 0; +} +*/ + +/* + ALLOC_ALIGN_SIZE >= sizeof(void *) + ALLOC_ALIGN_SIZE >= cache_line_size +*/ + +#define ALLOC_ALIGN_SIZE ((size_t)1 << 7) + +static void *SzAlignedAlloc(ISzAllocPtr pp, size_t size) +{ + #ifndef USE_posix_memalign + + void *p; + void *pAligned; + size_t newSize; + UNUSED_VAR(pp); + + /* also we can allocate additional dummy ALLOC_ALIGN_SIZE bytes after aligned + block to prevent cache line sharing with another allocated blocks */ + + newSize = size + ALLOC_ALIGN_SIZE * 1 + ADJUST_ALLOC_SIZE; + if (newSize < size) + return NULL; + + p = MyAlloc(newSize); + + if (!p) + return NULL; + pAligned = MY_ALIGN_PTR_UP_PLUS(p, ALLOC_ALIGN_SIZE); + + Print(" size="); PrintHex(size, 8); + Print(" a_size="); PrintHex(newSize, 8); + Print(" ptr="); PrintAddr(p); + Print(" a_ptr="); PrintAddr(pAligned); + PrintLn(); + + ((void **)pAligned)[-1] = p; + + return pAligned; + + #else + + void *p; + UNUSED_VAR(pp); + if (posix_memalign(&p, ALLOC_ALIGN_SIZE, size)) + return NULL; + + Print(" posix_memalign="); PrintAddr(p); + PrintLn(); + + return p; + + #endif +} + + +static void SzAlignedFree(ISzAllocPtr pp, void *address) +{ + UNUSED_VAR(pp); + #ifndef USE_posix_memalign + if (address) + MyFree(((void **)address)[-1]); + #else + free(address); + #endif +} + + +const ISzAlloc g_AlignedAlloc = { SzAlignedAlloc, SzAlignedFree }; + + + +#define MY_ALIGN_PTR_DOWN_1(p) MY_ALIGN_PTR_DOWN(p, sizeof(void *)) + +/* we align ptr to support cases where CAlignOffsetAlloc::offset is not multiply of sizeof(void *) */ +#define REAL_BLOCK_PTR_VAR(p) ((void **)MY_ALIGN_PTR_DOWN_1(p))[-1] +/* +#define REAL_BLOCK_PTR_VAR(p) ((void **)(p))[-1] +*/ + +static void *AlignOffsetAlloc_Alloc(ISzAllocPtr pp, size_t size) +{ + CAlignOffsetAlloc *p = CONTAINER_FROM_VTBL(pp, CAlignOffsetAlloc, vt); + void *adr; + void *pAligned; + size_t newSize; + size_t extra; + size_t alignSize = (size_t)1 << p->numAlignBits; + + if (alignSize < sizeof(void *)) + alignSize = sizeof(void *); + + if (p->offset >= alignSize) + return NULL; + + /* also we can allocate additional dummy ALLOC_ALIGN_SIZE bytes after aligned + block to prevent cache line sharing with another allocated blocks */ + extra = p->offset & (sizeof(void *) - 1); + newSize = size + alignSize + extra + ADJUST_ALLOC_SIZE; + if (newSize < size) + return NULL; + + adr = ISzAlloc_Alloc(p->baseAlloc, newSize); + + if (!adr) + return NULL; + + pAligned = (char *)MY_ALIGN_PTR_DOWN((char *)adr + + alignSize - p->offset + extra + ADJUST_ALLOC_SIZE, alignSize) + p->offset; + + PrintLn(); + Print("- Aligned: "); + Print(" size="); PrintHex(size, 8); + Print(" a_size="); PrintHex(newSize, 8); + Print(" ptr="); PrintAddr(adr); + Print(" a_ptr="); PrintAddr(pAligned); + PrintLn(); + + REAL_BLOCK_PTR_VAR(pAligned) = adr; + + return pAligned; +} + + +static void AlignOffsetAlloc_Free(ISzAllocPtr pp, void *address) +{ + if (address) + { + CAlignOffsetAlloc *p = CONTAINER_FROM_VTBL(pp, CAlignOffsetAlloc, vt); + PrintLn(); + Print("- Aligned Free: "); + PrintLn(); + ISzAlloc_Free(p->baseAlloc, REAL_BLOCK_PTR_VAR(address)); + } +} + + +void AlignOffsetAlloc_CreateVTable(CAlignOffsetAlloc *p) +{ + p->vt.Alloc = AlignOffsetAlloc_Alloc; + p->vt.Free = AlignOffsetAlloc_Free; +} diff --git a/C/Alloc.h b/C/Alloc.h index 52e309ff..3d796e5e 100644 --- a/C/Alloc.h +++ b/C/Alloc.h @@ -1,5 +1,5 @@ /* Alloc.h -- Memory allocation functions -2017-04-03 : Igor Pavlov : Public domain */ +2018-02-19 : Igor Pavlov : Public domain */ #ifndef __COMMON_ALLOC_H #define __COMMON_ALLOC_H @@ -31,6 +31,20 @@ void BigFree(void *address); extern const ISzAlloc g_Alloc; extern const ISzAlloc g_BigAlloc; +extern const ISzAlloc g_MidAlloc; +extern const ISzAlloc g_AlignedAlloc; + + +typedef struct +{ + ISzAlloc vt; + ISzAllocPtr baseAlloc; + unsigned numAlignBits; /* ((1 << numAlignBits) >= sizeof(void *)) */ + size_t offset; /* (offset == (k * sizeof(void *)) && offset < (1 << numAlignBits) */ +} CAlignOffsetAlloc; + +void AlignOffsetAlloc_CreateVTable(CAlignOffsetAlloc *p); + EXTERN_C_END diff --git a/C/Bcj2.c b/C/Bcj2.c index 2b15b604..da93985c 100644 --- a/C/Bcj2.c +++ b/C/Bcj2.c @@ -1,5 +1,5 @@ /* Bcj2.c -- BCJ2 Decoder (Converter for x86 code) -2017-04-03 : Igor Pavlov : Public domain */ +2018-04-28 : Igor Pavlov : Public domain */ #include "Precomp.h" @@ -232,10 +232,10 @@ SRes Bcj2Dec_Decode(CBcj2Dec *p) if (rem < 4) { - SizeT i; - SetUi32(p->temp, val); - for (i = 0; i < rem; i++) - dest[i] = p->temp[i]; + p->temp[0] = (Byte)val; if (rem > 0) dest[0] = (Byte)val; val >>= 8; + p->temp[1] = (Byte)val; if (rem > 1) dest[1] = (Byte)val; val >>= 8; + p->temp[2] = (Byte)val; if (rem > 2) dest[2] = (Byte)val; val >>= 8; + p->temp[3] = (Byte)val; p->dest = dest + rem; p->state = BCJ2_DEC_STATE_ORIG_0 + (unsigned)rem; break; diff --git a/C/Bcj2Enc.c b/C/Bcj2Enc.c index d5918734..b0bc7593 100644 --- a/C/Bcj2Enc.c +++ b/C/Bcj2Enc.c @@ -1,5 +1,5 @@ /* Bcj2Enc.c -- BCJ2 Encoder (Converter for x86 code) -2017-04-03 : Igor Pavlov : Public domain */ +2017-04-28 : Igor Pavlov : Public domain */ #include "Precomp.h" @@ -12,7 +12,6 @@ #define PRF(x) #endif -#include #include #include "Bcj2.h" diff --git a/C/CpuArch.h b/C/CpuArch.h index 998c2a7e..7fb27282 100644 --- a/C/CpuArch.h +++ b/C/CpuArch.h @@ -1,5 +1,5 @@ /* CpuArch.h -- CPU specific code -2017-06-30 : Igor Pavlov : Public domain */ +2017-09-04 : Igor Pavlov : Public domain */ #ifndef __CPU_ARCH_H #define __CPU_ARCH_H @@ -174,7 +174,7 @@ MY_CPU_LE_UNALIGN means that CPU is LITTLE ENDIAN and CPU supports unaligned mem #ifndef MY_CPU_NAME #ifdef MY_CPU_LE #define MY_CPU_NAME "LE" - #elif MY_CPU_BE + #elif defined(MY_CPU_BE) #define MY_CPU_NAME "BE" #else /* diff --git a/C/Lzma2Dec.c b/C/Lzma2Dec.c index 1d4af16b..57e7f346 100644 --- a/C/Lzma2Dec.c +++ b/C/Lzma2Dec.c @@ -1,5 +1,5 @@ /* Lzma2Dec.c -- LZMA2 Decoder -2017-04-03 : Igor Pavlov : Public domain */ +2018-02-19 : Igor Pavlov : Public domain */ /* #define SHOW_DEBUG_INFO */ @@ -14,28 +14,22 @@ #include "Lzma2Dec.h" /* -00000000 - EOS -00000001 U U - Uncompressed Reset Dic -00000010 U U - Uncompressed No Reset -100uuuuu U U P P - LZMA no reset -101uuuuu U U P P - LZMA reset state -110uuuuu U U P P S - LZMA reset state + new prop -111uuuuu U U P P S - LZMA reset state + new prop + reset dic +00000000 - End of data +00000001 U U - Uncompressed, reset dic, need reset state and set new prop +00000010 U U - Uncompressed, no reset +100uuuuu U U P P - LZMA, no reset +101uuuuu U U P P - LZMA, reset state +110uuuuu U U P P S - LZMA, reset state + set new prop +111uuuuu U U P P S - LZMA, reset state + set new prop, reset dic u, U - Unpack Size P - Pack Size S - Props */ -#define LZMA2_CONTROL_LZMA (1 << 7) -#define LZMA2_CONTROL_COPY_NO_RESET 2 #define LZMA2_CONTROL_COPY_RESET_DIC 1 -#define LZMA2_CONTROL_EOF 0 -#define LZMA2_IS_UNCOMPRESSED_STATE(p) (((p)->control & LZMA2_CONTROL_LZMA) == 0) - -#define LZMA2_GET_LZMA_MODE(p) (((p)->control >> 5) & 3) -#define LZMA2_IS_THERE_PROP(mode) ((mode) >= 2) +#define LZMA2_IS_UNCOMPRESSED_STATE(p) (((p)->control & (1 << 7)) == 0) #define LZMA2_LCLP_MAX 4 #define LZMA2_DIC_SIZE_FROM_PROP(p) (((UInt32)2 | ((p) & 1)) << ((p) / 2 + 11)) @@ -91,9 +85,11 @@ SRes Lzma2Dec_Allocate(CLzma2Dec *p, Byte prop, ISzAllocPtr alloc) void Lzma2Dec_Init(CLzma2Dec *p) { p->state = LZMA2_STATE_CONTROL; - p->needInitDic = True; - p->needInitState = True; - p->needInitProp = True; + p->needInitLevel = 0xE0; + p->isExtraMode = False; + p->unpackSize = 0; + + // p->decoder.dicPos = 0; // we can use it instead of full init LzmaDec_Init(&p->decoder); } @@ -102,19 +98,26 @@ static ELzma2State Lzma2Dec_UpdateState(CLzma2Dec *p, Byte b) switch (p->state) { case LZMA2_STATE_CONTROL: + p->isExtraMode = False; p->control = b; - PRF(printf("\n %4X ", (unsigned)p->decoder.dicPos)); - PRF(printf(" %2X", (unsigned)b)); + PRF(printf("\n %8X", (unsigned)p->decoder.dicPos)); + PRF(printf(" %02X", (unsigned)b)); if (b == 0) return LZMA2_STATE_FINISHED; if (LZMA2_IS_UNCOMPRESSED_STATE(p)) { - if (b > 2) + if (b == LZMA2_CONTROL_COPY_RESET_DIC) + p->needInitLevel = 0xC0; + else if (b > 2 || p->needInitLevel == 0xE0) return LZMA2_STATE_ERROR; - p->unpackSize = 0; } else + { + if (b < p->needInitLevel) + return LZMA2_STATE_ERROR; + p->needInitLevel = 0; p->unpackSize = (UInt32)(b & 0x1F) << 16; + } return LZMA2_STATE_UNPACK0; case LZMA2_STATE_UNPACK0: @@ -124,8 +127,8 @@ static ELzma2State Lzma2Dec_UpdateState(CLzma2Dec *p, Byte b) case LZMA2_STATE_UNPACK1: p->unpackSize |= (UInt32)b; p->unpackSize++; - PRF(printf(" %8u", (unsigned)p->unpackSize)); - return (LZMA2_IS_UNCOMPRESSED_STATE(p)) ? LZMA2_STATE_DATA : LZMA2_STATE_PACK0; + PRF(printf(" %7u", (unsigned)p->unpackSize)); + return LZMA2_IS_UNCOMPRESSED_STATE(p) ? LZMA2_STATE_DATA : LZMA2_STATE_PACK0; case LZMA2_STATE_PACK0: p->packSize = (UInt32)b << 8; @@ -134,9 +137,9 @@ static ELzma2State Lzma2Dec_UpdateState(CLzma2Dec *p, Byte b) case LZMA2_STATE_PACK1: p->packSize |= (UInt32)b; p->packSize++; - PRF(printf(" %8u", (unsigned)p->packSize)); - return LZMA2_IS_THERE_PROP(LZMA2_GET_LZMA_MODE(p)) ? LZMA2_STATE_PROP: - (p->needInitProp ? LZMA2_STATE_ERROR : LZMA2_STATE_DATA); + // if (p->packSize < 5) return LZMA2_STATE_ERROR; + PRF(printf(" %5u", (unsigned)p->packSize)); + return (p->control & 0x40) ? LZMA2_STATE_PROP : LZMA2_STATE_DATA; case LZMA2_STATE_PROP: { @@ -145,13 +148,12 @@ static ELzma2State Lzma2Dec_UpdateState(CLzma2Dec *p, Byte b) return LZMA2_STATE_ERROR; lc = b % 9; b /= 9; - p->decoder.prop.pb = b / 5; + p->decoder.prop.pb = (Byte)(b / 5); lp = b % 5; if (lc + lp > LZMA2_LCLP_MAX) return LZMA2_STATE_ERROR; - p->decoder.prop.lc = lc; - p->decoder.prop.lp = lp; - p->needInitProp = False; + p->decoder.prop.lc = (Byte)lc; + p->decoder.prop.lp = (Byte)lp; return LZMA2_STATE_DATA; } } @@ -231,11 +233,6 @@ SRes Lzma2Dec_DecodeToDic(CLzma2Dec *p, SizeT dicLimit, if (p->state == LZMA2_STATE_DATA) { Bool initDic = (p->control == LZMA2_CONTROL_COPY_RESET_DIC); - if (initDic) - p->needInitProp = p->needInitState = True; - else if (p->needInitDic) - break; - p->needInitDic = False; LzmaDec_InitDicAndState(&p->decoder, initDic, False); } @@ -257,23 +254,17 @@ SRes Lzma2Dec_DecodeToDic(CLzma2Dec *p, SizeT dicLimit, if (p->state == LZMA2_STATE_DATA) { - unsigned mode = LZMA2_GET_LZMA_MODE(p); - Bool initDic = (mode == 3); - Bool initState = (mode != 0); - if ((!initDic && p->needInitDic) || (!initState && p->needInitState)) - break; - + Bool initDic = (p->control >= 0xE0); + Bool initState = (p->control >= 0xA0); LzmaDec_InitDicAndState(&p->decoder, initDic, initState); - p->needInitDic = False; - p->needInitState = False; p->state = LZMA2_STATE_DATA_CONT; } if (inCur > p->packSize) inCur = (SizeT)p->packSize; - - res = LzmaDec_DecodeToDic(&p->decoder, dicPos + outCur, src, &inCur, curFinishMode, status); + res = LzmaDec_DecodeToDic(&p->decoder, dicPos + outCur, src, &inCur, curFinishMode, status); + src += inCur; *srcLen += inCur; p->packSize -= (UInt32)inCur; @@ -310,6 +301,129 @@ SRes Lzma2Dec_DecodeToDic(CLzma2Dec *p, SizeT dicLimit, } + + +ELzma2ParseStatus Lzma2Dec_Parse(CLzma2Dec *p, + SizeT outSize, + const Byte *src, SizeT *srcLen, + int checkFinishBlock) +{ + SizeT inSize = *srcLen; + *srcLen = 0; + + while (p->state != LZMA2_STATE_ERROR) + { + if (p->state == LZMA2_STATE_FINISHED) + return LZMA_STATUS_FINISHED_WITH_MARK; + + if (outSize == 0 && !checkFinishBlock) + return LZMA_STATUS_NOT_FINISHED; + + if (p->state != LZMA2_STATE_DATA && p->state != LZMA2_STATE_DATA_CONT) + { + if (*srcLen == inSize) + return LZMA_STATUS_NEEDS_MORE_INPUT; + (*srcLen)++; + + p->state = Lzma2Dec_UpdateState(p, *src++); + + if (p->state == LZMA2_STATE_UNPACK0) + { + // if (p->decoder.dicPos != 0) + if (p->control == LZMA2_CONTROL_COPY_RESET_DIC || p->control >= 0xE0) + return LZMA2_PARSE_STATUS_NEW_BLOCK; + // if (outSize == 0) return LZMA_STATUS_NOT_FINISHED; + } + + // The following code can be commented. + // It's not big problem, if we read additional input bytes. + // It will be stopped later in LZMA2_STATE_DATA / LZMA2_STATE_DATA_CONT state. + + if (outSize == 0 && p->state != LZMA2_STATE_FINISHED) + { + // checkFinishBlock is true. So we expect that block must be finished, + // We can return LZMA_STATUS_NOT_SPECIFIED or LZMA_STATUS_NOT_FINISHED here + // break; + return LZMA_STATUS_NOT_FINISHED; + } + + if (p->state == LZMA2_STATE_DATA) + return LZMA2_PARSE_STATUS_NEW_CHUNK; + + continue; + } + + if (outSize == 0) + return LZMA_STATUS_NOT_FINISHED; + + { + SizeT inCur = inSize - *srcLen; + + if (LZMA2_IS_UNCOMPRESSED_STATE(p)) + { + if (inCur == 0) + return LZMA_STATUS_NEEDS_MORE_INPUT; + if (inCur > p->unpackSize) + inCur = p->unpackSize; + if (inCur > outSize) + inCur = outSize; + p->decoder.dicPos += inCur; + src += inCur; + *srcLen += inCur; + outSize -= inCur; + p->unpackSize -= (UInt32)inCur; + p->state = (p->unpackSize == 0) ? LZMA2_STATE_CONTROL : LZMA2_STATE_DATA_CONT; + } + else + { + p->isExtraMode = True; + + if (inCur == 0) + { + if (p->packSize != 0) + return LZMA_STATUS_NEEDS_MORE_INPUT; + } + else if (p->state == LZMA2_STATE_DATA) + { + p->state = LZMA2_STATE_DATA_CONT; + if (*src != 0) + { + // first byte of lzma chunk must be Zero + *srcLen += 1; + p->packSize--; + break; + } + } + + if (inCur > p->packSize) + inCur = (SizeT)p->packSize; + + src += inCur; + *srcLen += inCur; + p->packSize -= (UInt32)inCur; + + if (p->packSize == 0) + { + SizeT rem = outSize; + if (rem > p->unpackSize) + rem = p->unpackSize; + p->decoder.dicPos += rem; + p->unpackSize -= (UInt32)rem; + outSize -= rem; + if (p->unpackSize == 0) + p->state = LZMA2_STATE_CONTROL; + } + } + } + } + + p->state = LZMA2_STATE_ERROR; + return LZMA_STATUS_NOT_SPECIFIED; +} + + + + SRes Lzma2Dec_DecodeToBuf(CLzma2Dec *p, Byte *dest, SizeT *destLen, const Byte *src, SizeT *srcLen, ELzmaFinishMode finishMode, ELzmaStatus *status) { SizeT outSize = *destLen, inSize = *srcLen; diff --git a/C/Lzma2Dec.h b/C/Lzma2Dec.h index 8c3de753..da503872 100644 --- a/C/Lzma2Dec.h +++ b/C/Lzma2Dec.h @@ -1,5 +1,5 @@ /* Lzma2Dec.h -- LZMA2 Decoder -2017-04-03 : Igor Pavlov : Public domain */ +2018-02-19 : Igor Pavlov : Public domain */ #ifndef __LZMA2_DEC_H #define __LZMA2_DEC_H @@ -12,25 +12,24 @@ EXTERN_C_BEGIN typedef struct { - CLzmaDec decoder; - UInt32 packSize; - UInt32 unpackSize; unsigned state; Byte control; - Bool needInitDic; - Bool needInitState; - Bool needInitProp; + Byte needInitLevel; + Byte isExtraMode; + Byte _pad_; + UInt32 packSize; + UInt32 unpackSize; + CLzmaDec decoder; } CLzma2Dec; #define Lzma2Dec_Construct(p) LzmaDec_Construct(&(p)->decoder) -#define Lzma2Dec_FreeProbs(p, alloc) LzmaDec_FreeProbs(&(p)->decoder, alloc); -#define Lzma2Dec_Free(p, alloc) LzmaDec_Free(&(p)->decoder, alloc); +#define Lzma2Dec_FreeProbs(p, alloc) LzmaDec_FreeProbs(&(p)->decoder, alloc) +#define Lzma2Dec_Free(p, alloc) LzmaDec_Free(&(p)->decoder, alloc) SRes Lzma2Dec_AllocateProbs(CLzma2Dec *p, Byte prop, ISzAllocPtr alloc); SRes Lzma2Dec_Allocate(CLzma2Dec *p, Byte prop, ISzAllocPtr alloc); void Lzma2Dec_Init(CLzma2Dec *p); - /* finishMode: It has meaning only if the decoding reaches output limit (*destLen or dicLimit). @@ -53,6 +52,47 @@ SRes Lzma2Dec_DecodeToBuf(CLzma2Dec *p, Byte *dest, SizeT *destLen, const Byte *src, SizeT *srcLen, ELzmaFinishMode finishMode, ELzmaStatus *status); +/* ---------- LZMA2 block and chunk parsing ---------- */ + +/* +Lzma2Dec_Parse() parses compressed data stream up to next independent block or next chunk data. +It can return LZMA_STATUS_* code or LZMA2_PARSE_STATUS_* code: + - LZMA2_PARSE_STATUS_NEW_BLOCK - there is new block, and 1 additional byte (control byte of next block header) was read from input. + - LZMA2_PARSE_STATUS_NEW_CHUNK - there is new chunk, and only lzma2 header of new chunk was read. + CLzma2Dec::unpackSize contains unpack size of that chunk +*/ + +typedef enum +{ +/* + LZMA_STATUS_NOT_SPECIFIED // data error + LZMA_STATUS_FINISHED_WITH_MARK + LZMA_STATUS_NOT_FINISHED // + LZMA_STATUS_NEEDS_MORE_INPUT + LZMA_STATUS_MAYBE_FINISHED_WITHOUT_MARK // unused +*/ + LZMA2_PARSE_STATUS_NEW_BLOCK = LZMA_STATUS_MAYBE_FINISHED_WITHOUT_MARK + 1, + LZMA2_PARSE_STATUS_NEW_CHUNK +} ELzma2ParseStatus; + +ELzma2ParseStatus Lzma2Dec_Parse(CLzma2Dec *p, + SizeT outSize, // output size + const Byte *src, SizeT *srcLen, + int checkFinishBlock // set (checkFinishBlock = 1), if it must read full input data, if decoder.dicPos reaches blockMax position. + ); + +/* +LZMA2 parser doesn't decode LZMA chunks, so we must read + full input LZMA chunk to decode some part of LZMA chunk. + +Lzma2Dec_GetUnpackExtra() returns the value that shows + max possible number of output bytes that can be output by decoder + at current input positon. +*/ + +#define Lzma2Dec_GetUnpackExtra(p) ((p)->isExtraMode ? (p)->unpackSize : 0); + + /* ---------- One Call Interface ---------- */ /* diff --git a/C/Lzma2DecMt.c b/C/Lzma2DecMt.c new file mode 100644 index 00000000..be698cbe --- /dev/null +++ b/C/Lzma2DecMt.c @@ -0,0 +1,1082 @@ +/* Lzma2DecMt.c -- LZMA2 Decoder Multi-thread +2018-03-02 : Igor Pavlov : Public domain */ + +#include "Precomp.h" + +// #define SHOW_DEBUG_INFO + +#ifdef SHOW_DEBUG_INFO +#include +#endif + +#ifdef SHOW_DEBUG_INFO +#define PRF(x) x +#else +#define PRF(x) +#endif + +#define PRF_STR(s) PRF(printf("\n" s "\n")) +#define PRF_STR_INT(s, d) PRF(printf("\n" s " %d\n", (unsigned)d)) +#define PRF_STR_INT_2(s, d1, d2) PRF(printf("\n" s " %d %d\n", (unsigned)d1, (unsigned)d2)) + +// #define _7ZIP_ST + +#include "Alloc.h" + +#include "Lzma2Dec.h" +#include "Lzma2DecMt.h" + +#ifndef _7ZIP_ST +#include "MtDec.h" +#endif + + +#define LZMA2DECMT_OUT_BLOCK_MAX_DEFAULT (1 << 28) + +void Lzma2DecMtProps_Init(CLzma2DecMtProps *p) +{ + p->inBufSize_ST = 1 << 20; + p->outStep_ST = 1 << 20; + + #ifndef _7ZIP_ST + p->numThreads = 1; + p->inBufSize_MT = 1 << 18; + p->outBlockMax = LZMA2DECMT_OUT_BLOCK_MAX_DEFAULT; + p->inBlockMax = p->outBlockMax + p->outBlockMax / 16; + #endif +} + + + +#ifndef _7ZIP_ST + +/* ---------- CLzma2DecMtThread ---------- */ + +typedef struct +{ + CLzma2Dec dec; + Byte dec_created; + Byte needInit; + + Byte *outBuf; + size_t outBufSize; + + EMtDecParseState state; + ELzma2ParseStatus parseStatus; + + size_t inPreSize; + size_t outPreSize; + + size_t inCodeSize; + size_t outCodeSize; + SRes codeRes; + + CAlignOffsetAlloc alloc; + + Byte mtPad[1 << 7]; +} CLzma2DecMtThread; + +#endif + + +/* ---------- CLzma2DecMt ---------- */ + +typedef struct +{ + // ISzAllocPtr alloc; + ISzAllocPtr allocMid; + + CAlignOffsetAlloc alignOffsetAlloc; + CLzma2DecMtProps props; + Byte prop; + + ISeqInStream *inStream; + ISeqOutStream *outStream; + ICompressProgress *progress; + + Bool finishMode; + Bool outSize_Defined; + UInt64 outSize; + + UInt64 outProcessed; + UInt64 inProcessed; + Bool readWasFinished; + SRes readRes; + + Byte *inBuf; + size_t inBufSize; + Byte dec_created; + CLzma2Dec dec; + + size_t inPos; + size_t inLim; + + #ifndef _7ZIP_ST + UInt64 outProcessed_Parse; + Bool mtc_WasConstructed; + CMtDec mtc; + CLzma2DecMtThread coders[MTDEC__THREADS_MAX]; + #endif + +} CLzma2DecMt; + + + +CLzma2DecMtHandle Lzma2DecMt_Create(ISzAllocPtr alloc, ISzAllocPtr allocMid) +{ + CLzma2DecMt *p = (CLzma2DecMt *)ISzAlloc_Alloc(alloc, sizeof(CLzma2DecMt)); + if (!p) + return NULL; + + // p->alloc = alloc; + p->allocMid = allocMid; + + AlignOffsetAlloc_CreateVTable(&p->alignOffsetAlloc); + p->alignOffsetAlloc.numAlignBits = 7; + p->alignOffsetAlloc.offset = 0; + p->alignOffsetAlloc.baseAlloc = alloc; + + p->inBuf = NULL; + p->inBufSize = 0; + p->dec_created = False; + + // Lzma2DecMtProps_Init(&p->props); + + #ifndef _7ZIP_ST + p->mtc_WasConstructed = False; + { + unsigned i; + for (i = 0; i < MTDEC__THREADS_MAX; i++) + { + CLzma2DecMtThread *t = &p->coders[i]; + t->dec_created = False; + t->outBuf = NULL; + t->outBufSize = 0; + } + } + #endif + + return p; +} + + +#ifndef _7ZIP_ST + +static void Lzma2DecMt_FreeOutBufs(CLzma2DecMt *p) +{ + unsigned i; + for (i = 0; i < MTDEC__THREADS_MAX; i++) + { + CLzma2DecMtThread *t = &p->coders[i]; + if (t->outBuf) + { + ISzAlloc_Free(p->allocMid, t->outBuf); + t->outBuf = NULL; + t->outBufSize = 0; + } + } +} + +#endif + + +static void Lzma2DecMt_FreeSt(CLzma2DecMt *p) +{ + if (p->dec_created) + { + Lzma2Dec_Free(&p->dec, &p->alignOffsetAlloc.vt); + p->dec_created = False; + } + if (p->inBuf) + { + ISzAlloc_Free(p->allocMid, p->inBuf); + p->inBuf = NULL; + } + p->inBufSize = 0; +} + + +void Lzma2DecMt_Destroy(CLzma2DecMtHandle pp) +{ + CLzma2DecMt *p = (CLzma2DecMt *)pp; + + Lzma2DecMt_FreeSt(p); + + #ifndef _7ZIP_ST + + if (p->mtc_WasConstructed) + { + MtDec_Destruct(&p->mtc); + p->mtc_WasConstructed = False; + } + { + unsigned i; + for (i = 0; i < MTDEC__THREADS_MAX; i++) + { + CLzma2DecMtThread *t = &p->coders[i]; + if (t->dec_created) + { + // we don't need to free dict here + Lzma2Dec_FreeProbs(&t->dec, &t->alloc.vt); // p->alloc !!! + t->dec_created = False; + } + } + } + Lzma2DecMt_FreeOutBufs(p); + + #endif + + ISzAlloc_Free(p->alignOffsetAlloc.baseAlloc, pp); +} + + + +#ifndef _7ZIP_ST + +static void Lzma2DecMt_MtCallback_Parse(void *obj, unsigned coderIndex, CMtDecCallbackInfo *cc) +{ + CLzma2DecMt *me = (CLzma2DecMt *)obj; + CLzma2DecMtThread *t = &me->coders[coderIndex]; + + PRF_STR_INT_2("Parse", coderIndex, cc->srcSize); + + cc->state = MTDEC_PARSE_CONTINUE; + + if (cc->startCall) + { + if (!t->dec_created) + { + Lzma2Dec_Construct(&t->dec); + t->dec_created = True; + AlignOffsetAlloc_CreateVTable(&t->alloc); + { + /* (1 << 12) is expected size of one way in data cache. + We optimize alignment for cache line size of 128 bytes and smaller */ + const unsigned kNumAlignBits = 12; + const unsigned kNumCacheLineBits = 7; /* <= kNumAlignBits */ + t->alloc.numAlignBits = kNumAlignBits; + t->alloc.offset = ((UInt32)coderIndex * ((1 << 11) + (1 << 8) + (1 << 6))) & ((1 << kNumAlignBits) - (1 << kNumCacheLineBits)); + t->alloc.baseAlloc = me->alignOffsetAlloc.baseAlloc; + } + } + Lzma2Dec_Init(&t->dec); + + t->inPreSize = 0; + t->outPreSize = 0; + // t->blockWasFinished = False; + // t->finishedWithMark = False; + t->parseStatus = LZMA_STATUS_NOT_SPECIFIED; + t->state = MTDEC_PARSE_CONTINUE; + + t->inCodeSize = 0; + t->outCodeSize = 0; + t->codeRes = SZ_OK; + + // (cc->srcSize == 0) is allowed + } + + { + ELzma2ParseStatus status; + Bool overflow; + UInt32 unpackRem = 0; + + int checkFinishBlock = True; + size_t limit = me->props.outBlockMax; + if (me->outSize_Defined) + { + UInt64 rem = me->outSize - me->outProcessed_Parse; + if (limit >= rem) + { + limit = (size_t)rem; + if (!me->finishMode) + checkFinishBlock = False; + } + } + + // checkFinishBlock = False, if we want to decode partial data + // that must be finished at position <= outBlockMax. + + { + const SizeT srcOrig = cc->srcSize; + SizeT srcSize_Point = 0; + SizeT dicPos_Point = 0; + + cc->srcSize = 0; + overflow = False; + + for (;;) + { + SizeT srcCur = srcOrig - cc->srcSize; + + status = Lzma2Dec_Parse(&t->dec, + limit - t->dec.decoder.dicPos, + cc->src + cc->srcSize, &srcCur, + checkFinishBlock); + + cc->srcSize += srcCur; + + if (status == LZMA2_PARSE_STATUS_NEW_CHUNK) + { + if (t->dec.unpackSize > me->props.outBlockMax - t->dec.decoder.dicPos) + { + overflow = True; + break; + } + continue; + } + + if (status == LZMA2_PARSE_STATUS_NEW_BLOCK) + { + if (t->dec.decoder.dicPos == 0) + continue; + // we decode small blocks in one thread + if (t->dec.decoder.dicPos >= (1 << 14)) + break; + dicPos_Point = t->dec.decoder.dicPos; + srcSize_Point = cc->srcSize; + continue; + } + + if ((int)status == LZMA_STATUS_NOT_FINISHED && checkFinishBlock + // && limit == t->dec.decoder.dicPos + // && limit == me->props.outBlockMax + ) + { + overflow = True; + break; + } + + unpackRem = Lzma2Dec_GetUnpackExtra(&t->dec); + break; + } + + if (dicPos_Point != 0 + && (int)status != LZMA2_PARSE_STATUS_NEW_BLOCK + && (int)status != LZMA_STATUS_FINISHED_WITH_MARK + && (int)status != LZMA_STATUS_NOT_SPECIFIED) + { + // we revert to latest newBlock state + status = LZMA2_PARSE_STATUS_NEW_BLOCK; + unpackRem = 0; + t->dec.decoder.dicPos = dicPos_Point; + cc->srcSize = srcSize_Point; + overflow = False; + } + } + + t->inPreSize += cc->srcSize; + t->parseStatus = status; + + if (overflow) + cc->state = MTDEC_PARSE_OVERFLOW; + else + { + size_t dicPos = t->dec.decoder.dicPos; + + if ((int)status != LZMA_STATUS_NEEDS_MORE_INPUT) + { + if (status == LZMA2_PARSE_STATUS_NEW_BLOCK) + { + cc->state = MTDEC_PARSE_NEW; + cc->srcSize--; // we don't need control byte of next block + t->inPreSize--; + } + else + { + cc->state = MTDEC_PARSE_END; + if ((int)status != LZMA_STATUS_FINISHED_WITH_MARK) + { + // (status == LZMA_STATUS_NOT_SPECIFIED) + // (status == LZMA_STATUS_NOT_FINISHED) + if (unpackRem != 0) + { + /* we also reserve space for max possible number of output bytes of current LZMA chunk */ + SizeT rem = limit - dicPos; + if (rem > unpackRem) + rem = unpackRem; + dicPos += rem; + } + } + } + + me->outProcessed_Parse += dicPos; + } + + cc->outPos = dicPos; + t->outPreSize = (size_t)dicPos; + } + + t->state = cc->state; + return; + } +} + + +static SRes Lzma2DecMt_MtCallback_PreCode(void *pp, unsigned coderIndex) +{ + CLzma2DecMt *me = (CLzma2DecMt *)pp; + CLzma2DecMtThread *t = &me->coders[coderIndex]; + Byte *dest = t->outBuf; + + if (t->inPreSize == 0) + { + t->codeRes = SZ_ERROR_DATA; + return t->codeRes; + } + + if (!dest || t->outBufSize < t->outPreSize) + { + if (dest) + { + ISzAlloc_Free(me->allocMid, dest); + t->outBuf = NULL; + t->outBufSize = 0; + } + + dest = (Byte *)ISzAlloc_Alloc(me->allocMid, t->outPreSize + // + (1 << 28) + ); + // Sleep(200); + if (!dest) + return SZ_ERROR_MEM; + t->outBuf = dest; + t->outBufSize = t->outPreSize; + } + + t->dec.decoder.dic = dest; + t->dec.decoder.dicBufSize = t->outPreSize; + + t->needInit = True; + + return Lzma2Dec_AllocateProbs(&t->dec, me->prop, &t->alloc.vt); // alloc.vt +} + + +static SRes Lzma2DecMt_MtCallback_Code(void *pp, unsigned coderIndex, + const Byte *src, size_t srcSize, int srcFinished, + // int finished, int blockFinished, + UInt64 *inCodePos, UInt64 *outCodePos, int *stop) +{ + CLzma2DecMt *me = (CLzma2DecMt *)pp; + CLzma2DecMtThread *t = &me->coders[coderIndex]; + + UNUSED_VAR(srcFinished) + + PRF_STR_INT_2("Code", coderIndex, srcSize); + + *inCodePos = t->inCodeSize; + *outCodePos = 0; + *stop = True; + + if (t->needInit) + { + Lzma2Dec_Init(&t->dec); + t->needInit = False; + } + + { + ELzmaStatus status; + size_t srcProcessed = srcSize; + Bool blockWasFinished = + ((int)t->parseStatus == LZMA_STATUS_FINISHED_WITH_MARK + || t->parseStatus == LZMA2_PARSE_STATUS_NEW_BLOCK); + + SRes res = Lzma2Dec_DecodeToDic(&t->dec, + t->outPreSize, + src, &srcProcessed, + blockWasFinished ? LZMA_FINISH_END : LZMA_FINISH_ANY, + &status); + + t->codeRes = res; + + t->inCodeSize += srcProcessed; + *inCodePos = t->inCodeSize; + t->outCodeSize = t->dec.decoder.dicPos; + *outCodePos = t->dec.decoder.dicPos; + + if (res != SZ_OK) + return res; + + if (srcProcessed == srcSize) + *stop = False; + + if (blockWasFinished) + { + if (srcSize != srcProcessed) + return SZ_ERROR_FAIL; + + if (t->inPreSize == t->inCodeSize) + { + if (t->outPreSize != t->outCodeSize) + return SZ_ERROR_FAIL; + *stop = True; + } + } + else + { + if (t->outPreSize == t->outCodeSize) + *stop = True; + } + + return SZ_OK; + } +} + + +#define LZMA2DECMT_STREAM_WRITE_STEP (1 << 24) + +static SRes Lzma2DecMt_MtCallback_Write(void *pp, unsigned coderIndex, + Bool needWriteToStream, + const Byte *src, size_t srcSize, + Bool *needContinue, Bool *canRecode) +{ + CLzma2DecMt *me = (CLzma2DecMt *)pp; + const CLzma2DecMtThread *t = &me->coders[coderIndex]; + size_t size = t->outCodeSize; + const Byte *data = t->outBuf; + Bool needContinue2 = True; + + PRF_STR_INT_2("Write", coderIndex, srcSize); + + *needContinue = False; + *canRecode = True; + UNUSED_VAR(src) + UNUSED_VAR(srcSize) + + if ( + // t->parseStatus == LZMA_STATUS_FINISHED_WITH_MARK + t->state == MTDEC_PARSE_OVERFLOW + || t->state == MTDEC_PARSE_END) + needContinue2 = False; + + + if (!needWriteToStream) + return SZ_OK; + + me->mtc.inProcessed += t->inCodeSize; + + if (t->codeRes == SZ_OK) + if ((int)t->parseStatus == LZMA_STATUS_FINISHED_WITH_MARK + || t->parseStatus == LZMA2_PARSE_STATUS_NEW_BLOCK) + if (t->outPreSize != t->outCodeSize + || t->inPreSize != t->inCodeSize) + return SZ_ERROR_FAIL; + + *canRecode = False; + + if (me->outStream) + { + for (;;) + { + size_t cur = size; + size_t written; + if (cur > LZMA2DECMT_STREAM_WRITE_STEP) + cur = LZMA2DECMT_STREAM_WRITE_STEP; + + written = ISeqOutStream_Write(me->outStream, data, cur); + + me->outProcessed += written; + // me->mtc.writtenTotal += written; + if (written != cur) + return SZ_ERROR_WRITE; + data += cur; + size -= cur; + if (size == 0) + { + *needContinue = needContinue2; + return SZ_OK; + } + RINOK(MtProgress_ProgressAdd(&me->mtc.mtProgress, 0, 0)); + } + } + + return SZ_ERROR_FAIL; + /* + if (size > me->outBufSize) + return SZ_ERROR_OUTPUT_EOF; + memcpy(me->outBuf, data, size); + me->outBufSize -= size; + me->outBuf += size; + *needContinue = needContinue2; + return SZ_OK; + */ +} + +#endif + + +static SRes Lzma2Dec_Prepare_ST(CLzma2DecMt *p) +{ + if (!p->dec_created) + { + Lzma2Dec_Construct(&p->dec); + p->dec_created = True; + } + + RINOK(Lzma2Dec_Allocate(&p->dec, p->prop, &p->alignOffsetAlloc.vt)); + + if (!p->inBuf || p->inBufSize != p->props.inBufSize_ST) + { + ISzAlloc_Free(p->allocMid, p->inBuf); + p->inBufSize = 0; + p->inBuf = (Byte *)ISzAlloc_Alloc(p->allocMid, p->props.inBufSize_ST); + if (!p->inBuf) + return SZ_ERROR_MEM; + p->inBufSize = p->props.inBufSize_ST; + } + + Lzma2Dec_Init(&p->dec); + + return SZ_OK; +} + + +static SRes Lzma2Dec_Decode_ST(CLzma2DecMt *p + #ifndef _7ZIP_ST + , Bool tMode + #endif + ) +{ + SizeT wrPos; + size_t inPos, inLim; + const Byte *inData; + UInt64 inPrev, outPrev; + + CLzma2Dec *dec; + + #ifndef _7ZIP_ST + if (tMode) + { + Lzma2DecMt_FreeOutBufs(p); + tMode = MtDec_PrepareRead(&p->mtc); + } + #endif + + RINOK(Lzma2Dec_Prepare_ST(p)); + + dec = &p->dec; + + inPrev = p->inProcessed; + outPrev = p->outProcessed; + + inPos = 0; + inLim = 0; + inData = NULL; + wrPos = dec->decoder.dicPos; + + for (;;) + { + SizeT dicPos; + SizeT size; + ELzmaFinishMode finishMode; + SizeT inProcessed; + ELzmaStatus status; + SRes res; + + SizeT outProcessed; + Bool outFinished; + Bool needStop; + + if (inPos == inLim) + { + #ifndef _7ZIP_ST + if (tMode) + { + inData = MtDec_Read(&p->mtc, &inLim); + inPos = 0; + if (inData) + continue; + tMode = False; + inLim = 0; + } + #endif + + if (!p->readWasFinished) + { + inPos = 0; + inLim = p->inBufSize; + inData = p->inBuf; + p->readRes = ISeqInStream_Read(p->inStream, (void *)inData, &inLim); + // p->readProcessed += inLim; + // inLim -= 5; p->readWasFinished = True; // for test + if (inLim == 0 || p->readRes != SZ_OK) + p->readWasFinished = True; + } + } + + dicPos = dec->decoder.dicPos; + { + SizeT next = dec->decoder.dicBufSize; + if (next - wrPos > p->props.outStep_ST) + next = wrPos + p->props.outStep_ST; + size = next - dicPos; + } + + finishMode = LZMA_FINISH_ANY; + if (p->outSize_Defined) + { + const UInt64 rem = p->outSize - p->outProcessed; + if (size >= rem) + { + size = (SizeT)rem; + if (p->finishMode) + finishMode = LZMA_FINISH_END; + } + } + + inProcessed = inLim - inPos; + + res = Lzma2Dec_DecodeToDic(dec, dicPos + size, inData + inPos, &inProcessed, finishMode, &status); + + inPos += inProcessed; + p->inProcessed += inProcessed; + outProcessed = dec->decoder.dicPos - dicPos; + p->outProcessed += outProcessed; + + outFinished = (p->outSize_Defined && p->outSize <= p->outProcessed); + + needStop = (res != SZ_OK + || (inProcessed == 0 && outProcessed == 0) + || status == LZMA_STATUS_FINISHED_WITH_MARK + || (!p->finishMode && outFinished)); + + if (needStop || outProcessed >= size) + { + SRes res2; + { + size_t writeSize = dec->decoder.dicPos - wrPos; + size_t written = ISeqOutStream_Write(p->outStream, dec->decoder.dic + wrPos, writeSize); + res2 = (written == writeSize) ? SZ_OK : SZ_ERROR_WRITE; + } + + if (dec->decoder.dicPos == dec->decoder.dicBufSize) + dec->decoder.dicPos = 0; + wrPos = dec->decoder.dicPos; + + RINOK(res2); + + if (needStop) + { + if (res != SZ_OK) + return res; + + if (status == LZMA_STATUS_FINISHED_WITH_MARK) + { + if (p->finishMode) + { + if (p->outSize_Defined && p->outSize != p->outProcessed) + return SZ_ERROR_DATA; + } + return SZ_OK; + } + + if (!p->finishMode && outFinished) + return SZ_OK; + + if (status == LZMA_STATUS_NEEDS_MORE_INPUT) + return SZ_ERROR_INPUT_EOF; + + return SZ_ERROR_DATA; + } + } + + if (p->progress) + { + UInt64 inDelta = p->inProcessed - inPrev; + UInt64 outDelta = p->outProcessed - outPrev; + if (inDelta >= (1 << 22) || outDelta >= (1 << 22)) + { + RINOK(ICompressProgress_Progress(p->progress, p->inProcessed, p->outProcessed)); + inPrev = p->inProcessed; + outPrev = p->outProcessed; + } + } + } +} + + + +SRes Lzma2DecMt_Decode(CLzma2DecMtHandle pp, + Byte prop, + const CLzma2DecMtProps *props, + ISeqOutStream *outStream, const UInt64 *outDataSize, int finishMode, + // Byte *outBuf, size_t *outBufSize, + ISeqInStream *inStream, + // const Byte *inData, size_t inDataSize, + UInt64 *inProcessed, + // UInt64 *outProcessed, + int *isMT, + ICompressProgress *progress) +{ + CLzma2DecMt *p = (CLzma2DecMt *)pp; + #ifndef _7ZIP_ST + Bool tMode; + #endif + + *inProcessed = 0; + + if (prop > 40) + return SZ_ERROR_UNSUPPORTED; + + p->prop = prop; + p->props = *props; + + p->inStream = inStream; + p->outStream = outStream; + p->progress = progress; + + p->outSize = 0; + p->outSize_Defined = False; + if (outDataSize) + { + p->outSize_Defined = True; + p->outSize = *outDataSize; + } + p->finishMode = finishMode; + + p->outProcessed = 0; + p->inProcessed = 0; + + p->readWasFinished = False; + + *isMT = False; + + + #ifndef _7ZIP_ST + + tMode = False; + + // p->mtc.parseRes = SZ_OK; + + // p->mtc.numFilledThreads = 0; + // p->mtc.crossStart = 0; + // p->mtc.crossEnd = 0; + // p->mtc.allocError_for_Read_BlockIndex = 0; + // p->mtc.isAllocError = False; + + if (p->props.numThreads > 1) + { + IMtDecCallback vt; + + Lzma2DecMt_FreeSt(p); + + p->outProcessed_Parse = 0; + + if (!p->mtc_WasConstructed) + { + p->mtc_WasConstructed = True; + MtDec_Construct(&p->mtc); + } + + p->mtc.progress = progress; + p->mtc.inStream = inStream; + + // p->outBuf = NULL; + // p->outBufSize = 0; + /* + if (!outStream) + { + // p->outBuf = outBuf; + // p->outBufSize = *outBufSize; + // *outBufSize = 0; + return SZ_ERROR_PARAM; + } + */ + + // p->mtc.inBlockMax = p->props.inBlockMax; + p->mtc.alloc = &p->alignOffsetAlloc.vt; + // p->alignOffsetAlloc.baseAlloc; + // p->mtc.inData = inData; + // p->mtc.inDataSize = inDataSize; + p->mtc.mtCallback = &vt; + p->mtc.mtCallbackObject = p; + + p->mtc.inBufSize = p->props.inBufSize_MT; + + p->mtc.numThreadsMax = p->props.numThreads; + + *isMT = True; + + vt.Parse = Lzma2DecMt_MtCallback_Parse; + vt.PreCode = Lzma2DecMt_MtCallback_PreCode; + vt.Code = Lzma2DecMt_MtCallback_Code; + vt.Write = Lzma2DecMt_MtCallback_Write; + + { + Bool needContinue = False; + + SRes res = MtDec_Code(&p->mtc); + + /* + if (!outStream) + *outBufSize = p->outBuf - outBuf; + */ + + *inProcessed = p->mtc.inProcessed; + + needContinue = False; + + if (res == SZ_OK) + { + if (p->mtc.mtProgress.res != SZ_OK) + res = p->mtc.mtProgress.res; + else + needContinue = p->mtc.needContinue; + } + + if (!needContinue) + { + if (res == SZ_OK) + return p->mtc.readRes; + return res; + } + + tMode = True; + p->readRes = p->mtc.readRes; + p->readWasFinished = p->mtc.readWasFinished; + p->inProcessed = p->mtc.inProcessed; + + PRF_STR("----- decoding ST -----"); + } + } + + #endif + + + *isMT = False; + + { + SRes res = Lzma2Dec_Decode_ST(p + #ifndef _7ZIP_ST + , tMode + #endif + ); + + *inProcessed = p->inProcessed; + + // res = SZ_OK; // for test + if (res == SZ_OK && p->readRes != SZ_OK) + res = p->readRes; + + /* + #ifndef _7ZIP_ST + if (res == SZ_OK && tMode && p->mtc.parseRes != SZ_OK) + res = p->mtc.parseRes; + #endif + */ + + return res; + } +} + + +/* ---------- Read from CLzma2DecMtHandle Interface ---------- */ + +SRes Lzma2DecMt_Init(CLzma2DecMtHandle pp, + Byte prop, + const CLzma2DecMtProps *props, + const UInt64 *outDataSize, int finishMode, + ISeqInStream *inStream) +{ + CLzma2DecMt *p = (CLzma2DecMt *)pp; + + if (prop > 40) + return SZ_ERROR_UNSUPPORTED; + + p->prop = prop; + p->props = *props; + + p->inStream = inStream; + + p->outSize = 0; + p->outSize_Defined = False; + if (outDataSize) + { + p->outSize_Defined = True; + p->outSize = *outDataSize; + } + p->finishMode = finishMode; + + p->outProcessed = 0; + p->inProcessed = 0; + + p->inPos = 0; + p->inLim = 0; + + return Lzma2Dec_Prepare_ST(p); +} + + +SRes Lzma2DecMt_Read(CLzma2DecMtHandle pp, + Byte *data, size_t *outSize, + UInt64 *inStreamProcessed) +{ + CLzma2DecMt *p = (CLzma2DecMt *)pp; + ELzmaFinishMode finishMode; + SRes readRes; + size_t size = *outSize; + + *outSize = 0; + *inStreamProcessed = 0; + + finishMode = LZMA_FINISH_ANY; + if (p->outSize_Defined) + { + const UInt64 rem = p->outSize - p->outProcessed; + if (size >= rem) + { + size = (size_t)rem; + if (p->finishMode) + finishMode = LZMA_FINISH_END; + } + } + + readRes = SZ_OK; + + for (;;) + { + SizeT inCur; + SizeT outCur; + ELzmaStatus status; + SRes res; + + if (p->inPos == p->inLim && readRes == SZ_OK) + { + p->inPos = 0; + p->inLim = p->props.inBufSize_ST; + readRes = ISeqInStream_Read(p->inStream, p->inBuf, &p->inLim); + } + + inCur = p->inLim - p->inPos; + outCur = size; + + res = Lzma2Dec_DecodeToBuf(&p->dec, data, &outCur, + p->inBuf + p->inPos, &inCur, finishMode, &status); + + p->inPos += inCur; + p->inProcessed += inCur; + *inStreamProcessed += inCur; + p->outProcessed += outCur; + *outSize += outCur; + size -= outCur; + data += outCur; + + if (res != 0) + return res; + + /* + if (status == LZMA_STATUS_FINISHED_WITH_MARK) + return readRes; + + if (size == 0 && status != LZMA_STATUS_NEEDS_MORE_INPUT) + { + if (p->finishMode && p->outSize_Defined && p->outProcessed >= p->outSize) + return SZ_ERROR_DATA; + return readRes; + } + */ + + if (inCur == 0 && outCur == 0) + return readRes; + } +} diff --git a/C/Lzma2DecMt.h b/C/Lzma2DecMt.h new file mode 100644 index 00000000..96f89a3c --- /dev/null +++ b/C/Lzma2DecMt.h @@ -0,0 +1,79 @@ +/* Lzma2DecMt.h -- LZMA2 Decoder Multi-thread +2018-02-17 : Igor Pavlov : Public domain */ + +#ifndef __LZMA2_DEC_MT_H +#define __LZMA2_DEC_MT_H + +#include "7zTypes.h" + +EXTERN_C_BEGIN + +typedef struct +{ + size_t inBufSize_ST; + size_t outStep_ST; + + #ifndef _7ZIP_ST + unsigned numThreads; + size_t inBufSize_MT; + size_t outBlockMax; + size_t inBlockMax; + #endif +} CLzma2DecMtProps; + +/* init to single-thread mode */ +void Lzma2DecMtProps_Init(CLzma2DecMtProps *p); + + +/* ---------- CLzma2DecMtHandle Interface ---------- */ + +/* Lzma2DecMt_ * functions can return the following exit codes: +SRes: + SZ_OK - OK + SZ_ERROR_MEM - Memory allocation error + SZ_ERROR_PARAM - Incorrect paramater in props + SZ_ERROR_WRITE - ISeqOutStream write callback error + // SZ_ERROR_OUTPUT_EOF - output buffer overflow - version with (Byte *) output + SZ_ERROR_PROGRESS - some break from progress callback + SZ_ERROR_THREAD - error in multithreading functions (only for Mt version) +*/ + +typedef void * CLzma2DecMtHandle; + +CLzma2DecMtHandle Lzma2DecMt_Create(ISzAllocPtr alloc, ISzAllocPtr allocMid); +void Lzma2DecMt_Destroy(CLzma2DecMtHandle p); + +SRes Lzma2DecMt_Decode(CLzma2DecMtHandle p, + Byte prop, + const CLzma2DecMtProps *props, + ISeqOutStream *outStream, + const UInt64 *outDataSize, // NULL means undefined + int finishMode, // 0 - partial unpacking is allowed, 1 - if lzma2 stream must be finished + // Byte *outBuf, size_t *outBufSize, + ISeqInStream *inStream, + // const Byte *inData, size_t inDataSize, + + // out variables: + UInt64 *inProcessed, + int *isMT, /* out: (*isMT == 0), if single thread decoding was used */ + + // UInt64 *outProcessed, + ICompressProgress *progress); + + +/* ---------- Read from CLzma2DecMtHandle Interface ---------- */ + +SRes Lzma2DecMt_Init(CLzma2DecMtHandle pp, + Byte prop, + const CLzma2DecMtProps *props, + const UInt64 *outDataSize, int finishMode, + ISeqInStream *inStream); + +SRes Lzma2DecMt_Read(CLzma2DecMtHandle pp, + Byte *data, size_t *outSize, + UInt64 *inStreamProcessed); + + +EXTERN_C_END + +#endif diff --git a/C/Lzma2Enc.c b/C/Lzma2Enc.c index 25e54af5..5098195c 100644 --- a/C/Lzma2Enc.c +++ b/C/Lzma2Enc.c @@ -1,5 +1,5 @@ /* Lzma2Enc.c -- LZMA2 Encoder -2017-08-28 : Igor Pavlov : Public domain */ +2018-04-27 : Igor Pavlov : Public domain */ #include "Precomp.h" @@ -369,7 +369,9 @@ typedef struct ISeqOutStream *outStream; Byte *outBuf; - size_t outBufSize; + size_t outBuf_Rem; /* remainder in outBuf */ + + size_t outBufSize; /* size of allocated outBufs[i] */ size_t outBufsDataSizes[MTCODER__BLOCKS_MAX]; Bool mtCoder_WasConstructed; CMtCoder mtCoder; @@ -666,7 +668,7 @@ static SRes Lzma2Enc_MtCallback_Code(void *pp, unsigned coderIndex, unsigned out if (!dest) { - dest = ISzAlloc_Alloc(me->alloc, me->outBufSize); + dest = (Byte *)ISzAlloc_Alloc(me->alloc, me->outBufSize); if (!dest) return SZ_ERROR_MEM; me->outBufs[outBufIndex] = dest; @@ -674,7 +676,8 @@ static SRes Lzma2Enc_MtCallback_Code(void *pp, unsigned coderIndex, unsigned out MtProgressThunk_CreateVTable(&progressThunk); progressThunk.mtProgress = &me->mtCoder.mtProgress; - progressThunk.index = coderIndex; + progressThunk.inSize = 0; + progressThunk.outSize = 0; res = Lzma2Enc_EncodeMt1(me, &me->coders[coderIndex], @@ -698,10 +701,10 @@ static SRes Lzma2Enc_MtCallback_Write(void *pp, unsigned outBufIndex) if (me->outStream) return ISeqOutStream_Write(me->outStream, data, size) == size ? SZ_OK : SZ_ERROR_WRITE; - if (size > me->outBufSize) + if (size > me->outBuf_Rem) return SZ_ERROR_OUTPUT_EOF; memcpy(me->outBuf, data, size); - me->outBufSize -= size; + me->outBuf_Rem -= size; me->outBuf += size; return SZ_OK; } @@ -720,10 +723,10 @@ SRes Lzma2Enc_Encode2(CLzma2EncHandle pp, CLzma2Enc *p = (CLzma2Enc *)pp; if (inStream && inData) - return E_INVALIDARG; + return SZ_ERROR_PARAM; if (outStream && outBuf) - return E_INVALIDARG; + return SZ_ERROR_PARAM; { unsigned i; @@ -748,11 +751,11 @@ SRes Lzma2Enc_Encode2(CLzma2EncHandle pp, p->outStream = outStream; p->outBuf = NULL; - p->outBufSize = 0; + p->outBuf_Rem = 0; if (!outStream) { p->outBuf = outBuf; - p->outBufSize = *outBufSize; + p->outBuf_Rem = *outBufSize; *outBufSize = 0; } diff --git a/C/LzmaDec.c b/C/LzmaDec.c index bd92d2dc..962b94bb 100644 --- a/C/LzmaDec.c +++ b/C/LzmaDec.c @@ -1,8 +1,9 @@ /* LzmaDec.c -- LZMA Decoder -2017-04-03 : Igor Pavlov : Public domain */ +2018-02-28 : Igor Pavlov : Public domain */ #include "Precomp.h" +/* #include "CpuArch.h" */ #include "LzmaDec.h" #include @@ -24,9 +25,16 @@ #define GET_BIT2(p, i, A0, A1) IF_BIT_0(p) \ { UPDATE_0(p); i = (i + i); A0; } else \ { UPDATE_1(p); i = (i + i) + 1; A1; } -#define GET_BIT(p, i) GET_BIT2(p, i, ; , ;) -#define TREE_GET_BIT(probs, i) { GET_BIT((probs + i), i); } +#define TREE_GET_BIT(probs, i) { GET_BIT2(probs + i, i, ;, ;); } + +#define REV_BIT(p, i, A0, A1) IF_BIT_0(p + i) \ + { UPDATE_0(p + i); A0; } else \ + { UPDATE_1(p + i); A1; } +#define REV_BIT_VAR( p, i, m) REV_BIT(p, i, i += m; m += m, m += m; i += m; ) +#define REV_BIT_CONST(p, i, m) REV_BIT(p, i, i += m; , i += m * 2; ) +#define REV_BIT_LAST( p, i, m) REV_BIT(p, i, i -= m , ; ) + #define TREE_DECODE(probs, limit, i) \ { i = 1; do { TREE_GET_BIT(probs, i); } while (i < limit); i -= limit; } @@ -46,12 +54,15 @@ i -= 0x40; } #endif -#define NORMAL_LITER_DEC GET_BIT(prob + symbol, symbol) +#define NORMAL_LITER_DEC TREE_GET_BIT(prob, symbol) #define MATCHED_LITER_DEC \ - matchByte <<= 1; \ - bit = (matchByte & offs); \ - probLit = prob + offs + bit + symbol; \ - GET_BIT2(probLit, symbol, offs &= ~bit, offs &= bit) + matchByte += matchByte; \ + bit = offs; \ + offs &= matchByte; \ + probLit = prob + (offs + bit + symbol); \ + GET_BIT2(probLit, symbol, offs ^= bit; , ;) + + #define NORMALIZE_CHECK if (range < kTopValue) { if (buf >= bufLimit) return DUMMY_ERROR; range <<= 8; code = (code << 8) | (*buf++); } @@ -66,25 +77,28 @@ { i = 1; do { GET_BIT_CHECK(probs + i, i) } while (i < limit); i -= limit; } +#define REV_BIT_CHECK(p, i, m) IF_BIT_0_CHECK(p + i) \ + { UPDATE_0_CHECK; i += m; m += m; } else \ + { UPDATE_1_CHECK; m += m; i += m; } + + #define kNumPosBitsMax 4 #define kNumPosStatesMax (1 << kNumPosBitsMax) #define kLenNumLowBits 3 #define kLenNumLowSymbols (1 << kLenNumLowBits) -#define kLenNumMidBits 3 -#define kLenNumMidSymbols (1 << kLenNumMidBits) #define kLenNumHighBits 8 #define kLenNumHighSymbols (1 << kLenNumHighBits) -#define LenChoice 0 -#define LenChoice2 (LenChoice + 1) -#define LenLow (LenChoice2 + 1) -#define LenMid (LenLow + (kNumPosStatesMax << kLenNumLowBits)) -#define LenHigh (LenMid + (kNumPosStatesMax << kLenNumMidBits)) +#define LenLow 0 +#define LenHigh (LenLow + 2 * (kNumPosStatesMax << kLenNumLowBits)) #define kNumLenProbs (LenHigh + kLenNumHighSymbols) +#define LenChoice LenLow +#define LenChoice2 (LenLow + (1 << kLenNumLowBits)) #define kNumStates 12 +#define kNumStates2 16 #define kNumLitStates 7 #define kStartPosModelIndex 4 @@ -98,54 +112,117 @@ #define kAlignTableSize (1 << kNumAlignBits) #define kMatchMinLen 2 -#define kMatchSpecLenStart (kMatchMinLen + kLenNumLowSymbols + kLenNumMidSymbols + kLenNumHighSymbols) +#define kMatchSpecLenStart (kMatchMinLen + kLenNumLowSymbols * 2 + kLenNumHighSymbols) -#define IsMatch 0 -#define IsRep (IsMatch + (kNumStates << kNumPosBitsMax)) +/* External ASM code needs same CLzmaProb array layout. So don't change it. */ + +/* (probs_1664) is faster and better for code size at some platforms */ +/* +#ifdef MY_CPU_X86_OR_AMD64 +*/ +#define kStartOffset 1664 +#define GET_PROBS p->probs_1664 +/* +#define GET_PROBS p->probs + kStartOffset +#else +#define kStartOffset 0 +#define GET_PROBS p->probs +#endif +*/ + +#define SpecPos (-kStartOffset) +#define IsRep0Long (SpecPos + kNumFullDistances) +#define RepLenCoder (IsRep0Long + (kNumStates2 << kNumPosBitsMax)) +#define LenCoder (RepLenCoder + kNumLenProbs) +#define IsMatch (LenCoder + kNumLenProbs) +#define Align (IsMatch + (kNumStates2 << kNumPosBitsMax)) +#define IsRep (Align + kAlignTableSize) #define IsRepG0 (IsRep + kNumStates) #define IsRepG1 (IsRepG0 + kNumStates) #define IsRepG2 (IsRepG1 + kNumStates) -#define IsRep0Long (IsRepG2 + kNumStates) -#define PosSlot (IsRep0Long + (kNumStates << kNumPosBitsMax)) -#define SpecPos (PosSlot + (kNumLenToPosStates << kNumPosSlotBits)) -#define Align (SpecPos + kNumFullDistances - kEndPosModelIndex) -#define LenCoder (Align + kAlignTableSize) -#define RepLenCoder (LenCoder + kNumLenProbs) -#define Literal (RepLenCoder + kNumLenProbs) +#define PosSlot (IsRepG2 + kNumStates) +#define Literal (PosSlot + (kNumLenToPosStates << kNumPosSlotBits)) +#define NUM_BASE_PROBS (Literal + kStartOffset) -#define LZMA_BASE_SIZE 1846 -#define LZMA_LIT_SIZE 0x300 - -#if Literal != LZMA_BASE_SIZE -StopCompilingDueBUG +#if Align != 0 && kStartOffset != 0 + #error Stop_Compiling_Bad_LZMA_kAlign #endif -#define LzmaProps_GetNumProbs(p) (Literal + ((UInt32)LZMA_LIT_SIZE << ((p)->lc + (p)->lp))) +#if NUM_BASE_PROBS != 1984 + #error Stop_Compiling_Bad_LZMA_PROBS +#endif + + +#define LZMA_LIT_SIZE 0x300 + +#define LzmaProps_GetNumProbs(p) (NUM_BASE_PROBS + ((UInt32)LZMA_LIT_SIZE << ((p)->lc + (p)->lp))) + + +#define CALC_POS_STATE(processedPos, pbMask) (((processedPos) & (pbMask)) << 4) +#define COMBINED_PS_STATE (posState + state) +#define GET_LEN_STATE (posState) #define LZMA_DIC_MIN (1 << 12) -/* First LZMA-symbol is always decoded. -And it decodes new LZMA-symbols while (buf < bufLimit), but "buf" is without last normalization +/* +p->remainLen : shows status of LZMA decoder: + < kMatchSpecLenStart : normal remain + = kMatchSpecLenStart : finished + = kMatchSpecLenStart + 1 : need init range coder + = kMatchSpecLenStart + 2 : need init range coder and state +*/ + +/* ---------- LZMA_DECODE_REAL ---------- */ +/* +LzmaDec_DecodeReal_3() can be implemented in external ASM file. +3 - is the code compatibility version of that function for check at link time. +*/ + +#define LZMA_DECODE_REAL LzmaDec_DecodeReal_3 + +/* +LZMA_DECODE_REAL() +In: + RangeCoder is normalized + if (p->dicPos == limit) + { + LzmaDec_TryDummy() was called before to exclude LITERAL and MATCH-REP cases. + So first symbol can be only MATCH-NON-REP. And if that MATCH-NON-REP symbol + is not END_OF_PAYALOAD_MARKER, then function returns error code. + } + +Processing: + first LZMA symbol will be decoded in any case + All checks for limits are at the end of main loop, + It will decode new LZMA-symbols while (p->buf < bufLimit && dicPos < limit), + RangeCoder is still without last normalization when (p->buf < bufLimit) is being checked. + Out: + RangeCoder is normalized Result: SZ_OK - OK SZ_ERROR_DATA - Error p->remainLen: < kMatchSpecLenStart : normal remain = kMatchSpecLenStart : finished - = kMatchSpecLenStart + 1 : Flush marker (unused now) - = kMatchSpecLenStart + 2 : State Init Marker (unused now) */ -static int MY_FAST_CALL LzmaDec_DecodeReal(CLzmaDec *p, SizeT limit, const Byte *bufLimit) -{ - CLzmaProb *probs = p->probs; - unsigned state = p->state; +#ifdef _LZMA_DEC_OPT + +int MY_FAST_CALL LZMA_DECODE_REAL(CLzmaDec *p, SizeT limit, const Byte *bufLimit); + +#else + +static +int MY_FAST_CALL LZMA_DECODE_REAL(CLzmaDec *p, SizeT limit, const Byte *bufLimit) +{ + CLzmaProb *probs = GET_PROBS; + unsigned state = (unsigned)p->state; UInt32 rep0 = p->reps[0], rep1 = p->reps[1], rep2 = p->reps[2], rep3 = p->reps[3]; unsigned pbMask = ((unsigned)1 << (p->prop.pb)) - 1; - unsigned lpMask = ((unsigned)1 << (p->prop.lp)) - 1; unsigned lc = p->prop.lc; + unsigned lpMask = ((unsigned)0x100 << p->prop.lp) - ((unsigned)0x100 >> lc); Byte *dic = p->dic; SizeT dicBufSize = p->dicBufSize; @@ -164,17 +241,16 @@ static int MY_FAST_CALL LzmaDec_DecodeReal(CLzmaDec *p, SizeT limit, const Byte CLzmaProb *prob; UInt32 bound; unsigned ttt; - unsigned posState = processedPos & pbMask; + unsigned posState = CALC_POS_STATE(processedPos, pbMask); - prob = probs + IsMatch + (state << kNumPosBitsMax) + posState; + prob = probs + IsMatch + COMBINED_PS_STATE; IF_BIT_0(prob) { unsigned symbol; UPDATE_0(prob); prob = probs + Literal; if (processedPos != 0 || checkDicSize != 0) - prob += ((UInt32)LZMA_LIT_SIZE * (((processedPos & lpMask) << lc) + - (dic[(dicPos == 0 ? dicBufSize : dicPos) - 1] >> (8 - lc)))); + prob += (UInt32)3 * ((((processedPos << 8) + dic[(dicPos == 0 ? dicBufSize : dicPos) - 1]) & lpMask) << lc); processedPos++; if (state < kNumLitStates) @@ -240,13 +316,16 @@ static int MY_FAST_CALL LzmaDec_DecodeReal(CLzmaDec *p, SizeT limit, const Byte else { UPDATE_1(prob); + /* + // that case was checked before with kBadRepCode if (checkDicSize == 0 && processedPos == 0) return SZ_ERROR_DATA; + */ prob = probs + IsRepG0 + state; IF_BIT_0(prob) { UPDATE_0(prob); - prob = probs + IsRep0Long + (state << kNumPosBitsMax) + posState; + prob = probs + IsRep0Long + COMBINED_PS_STATE; IF_BIT_0(prob) { UPDATE_0(prob); @@ -299,7 +378,7 @@ static int MY_FAST_CALL LzmaDec_DecodeReal(CLzmaDec *p, SizeT limit, const Byte IF_BIT_0(probLen) { UPDATE_0(probLen); - probLen = prob + LenLow + (posState << kLenNumLowBits); + probLen = prob + LenLow + GET_LEN_STATE; offset = 0; lim = (1 << kLenNumLowBits); } @@ -310,15 +389,15 @@ static int MY_FAST_CALL LzmaDec_DecodeReal(CLzmaDec *p, SizeT limit, const Byte IF_BIT_0(probLen) { UPDATE_0(probLen); - probLen = prob + LenMid + (posState << kLenNumMidBits); + probLen = prob + LenLow + GET_LEN_STATE + (1 << kLenNumLowBits); offset = kLenNumLowSymbols; - lim = (1 << kLenNumMidBits); + lim = (1 << kLenNumLowBits); } else { UPDATE_1(probLen); probLen = prob + LenHigh; - offset = kLenNumLowSymbols + kLenNumMidSymbols; + offset = kLenNumLowSymbols * 2; lim = (1 << kLenNumHighBits); } } @@ -331,7 +410,7 @@ static int MY_FAST_CALL LzmaDec_DecodeReal(CLzmaDec *p, SizeT limit, const Byte IF_BIT_0(probLen) { UPDATE_0(probLen); - probLen = prob + LenLow + (posState << kLenNumLowBits); + probLen = prob + LenLow + GET_LEN_STATE; len = 1; TREE_GET_BIT(probLen, len); TREE_GET_BIT(probLen, len); @@ -345,7 +424,7 @@ static int MY_FAST_CALL LzmaDec_DecodeReal(CLzmaDec *p, SizeT limit, const Byte IF_BIT_0(probLen) { UPDATE_0(probLen); - probLen = prob + LenMid + (posState << kLenNumMidBits); + probLen = prob + LenLow + GET_LEN_STATE + (1 << kLenNumLowBits); len = 1; TREE_GET_BIT(probLen, len); TREE_GET_BIT(probLen, len); @@ -356,7 +435,7 @@ static int MY_FAST_CALL LzmaDec_DecodeReal(CLzmaDec *p, SizeT limit, const Byte UPDATE_1(probLen); probLen = prob + LenHigh; TREE_DECODE(probLen, (1 << kLenNumHighBits), len); - len += kLenNumLowSymbols + kLenNumMidSymbols; + len += kLenNumLowSymbols * 2; } } } @@ -376,16 +455,16 @@ static int MY_FAST_CALL LzmaDec_DecodeReal(CLzmaDec *p, SizeT limit, const Byte if (posSlot < kEndPosModelIndex) { distance <<= numDirectBits; - prob = probs + SpecPos + distance - posSlot - 1; + prob = probs + SpecPos; { - UInt32 mask = 1; - unsigned i = 1; + UInt32 m = 1; + distance++; do { - GET_BIT2(prob + i, i, ; , distance |= mask); - mask <<= 1; + REV_BIT_VAR(prob, distance, m); } - while (--numDirectBits != 0); + while (--numDirectBits); + distance -= m; } } else @@ -412,19 +491,20 @@ static int MY_FAST_CALL LzmaDec_DecodeReal(CLzmaDec *p, SizeT limit, const Byte } */ } - while (--numDirectBits != 0); + while (--numDirectBits); prob = probs + Align; distance <<= kNumAlignBits; { unsigned i = 1; - GET_BIT2(prob + i, i, ; , distance |= 1); - GET_BIT2(prob + i, i, ; , distance |= 2); - GET_BIT2(prob + i, i, ; , distance |= 4); - GET_BIT2(prob + i, i, ; , distance |= 8); + REV_BIT_CONST(prob, i, 1); + REV_BIT_CONST(prob, i, 2); + REV_BIT_CONST(prob, i, 4); + REV_BIT_LAST (prob, i, 8); + distance |= i; } if (distance == (UInt32)0xFFFFFFFF) { - len += kMatchSpecLenStart; + len = kMatchSpecLenStart; state -= kNumStates; break; } @@ -435,20 +515,12 @@ static int MY_FAST_CALL LzmaDec_DecodeReal(CLzmaDec *p, SizeT limit, const Byte rep2 = rep1; rep1 = rep0; rep0 = distance + 1; - if (checkDicSize == 0) - { - if (distance >= processedPos) - { - p->dicPos = dicPos; - return SZ_ERROR_DATA; - } - } - else if (distance >= checkDicSize) + state = (state < kNumStates + kNumLitStates) ? kNumLitStates : kNumLitStates + 3; + if (distance >= (checkDicSize == 0 ? processedPos: checkDicSize)) { p->dicPos = dicPos; return SZ_ERROR_DATA; } - state = (state < kNumStates + kNumLitStates) ? kNumLitStates : kNumLitStates + 3; } len += kMatchMinLen; @@ -511,6 +583,7 @@ static int MY_FAST_CALL LzmaDec_DecodeReal(CLzmaDec *p, SizeT limit, const Byte return SZ_OK; } +#endif static void MY_FAST_CALL LzmaDec_WriteRem(CLzmaDec *p, SizeT limit) { @@ -519,7 +592,7 @@ static void MY_FAST_CALL LzmaDec_WriteRem(CLzmaDec *p, SizeT limit) Byte *dic = p->dic; SizeT dicPos = p->dicPos; SizeT dicBufSize = p->dicBufSize; - unsigned len = p->remainLen; + unsigned len = (unsigned)p->remainLen; SizeT rep0 = p->reps[0]; /* we use SizeT to avoid the BUG of VC14 for AMD64 */ SizeT rem = limit - dicPos; if (rem < len) @@ -540,6 +613,14 @@ static void MY_FAST_CALL LzmaDec_WriteRem(CLzmaDec *p, SizeT limit) } } + +#define kRange0 0xFFFFFFFF +#define kBound0 ((kRange0 >> kNumBitModelTotalBits) << (kNumBitModelTotalBits - 1)) +#define kBadRepCode (kBound0 + (((kRange0 - kBound0) >> kNumBitModelTotalBits) << (kNumBitModelTotalBits - 1))) +#if kBadRepCode != (0xC0000000 - 0x400) + #error Stop_Compiling_Bad_LZMA_Check +#endif + static int MY_FAST_CALL LzmaDec_DecodeReal2(CLzmaDec *p, SizeT limit, const Byte *bufLimit) { do @@ -550,9 +631,13 @@ static int MY_FAST_CALL LzmaDec_DecodeReal2(CLzmaDec *p, SizeT limit, const Byte UInt32 rem = p->prop.dicSize - p->processedPos; if (limit - p->dicPos > rem) limit2 = p->dicPos + rem; + + if (p->processedPos == 0) + if (p->code >= kBadRepCode) + return SZ_ERROR_DATA; } - - RINOK(LzmaDec_DecodeReal(p, limit2, bufLimit)); + + RINOK(LZMA_DECODE_REAL(p, limit2, bufLimit)); if (p->checkDicSize == 0 && p->processedPos >= p->prop.dicSize) p->checkDicSize = p->prop.dicSize; @@ -561,9 +646,6 @@ static int MY_FAST_CALL LzmaDec_DecodeReal2(CLzmaDec *p, SizeT limit, const Byte } while (p->dicPos < limit && p->buf < bufLimit && p->remainLen < kMatchSpecLenStart); - if (p->remainLen > kMatchSpecLenStart) - p->remainLen = kMatchSpecLenStart; - return 0; } @@ -580,17 +662,17 @@ static ELzmaDummy LzmaDec_TryDummy(const CLzmaDec *p, const Byte *buf, SizeT inS UInt32 range = p->range; UInt32 code = p->code; const Byte *bufLimit = buf + inSize; - const CLzmaProb *probs = p->probs; - unsigned state = p->state; + const CLzmaProb *probs = GET_PROBS; + unsigned state = (unsigned)p->state; ELzmaDummy res; { const CLzmaProb *prob; UInt32 bound; unsigned ttt; - unsigned posState = (p->processedPos) & ((1 << p->prop.pb) - 1); + unsigned posState = CALC_POS_STATE(p->processedPos, (1 << p->prop.pb) - 1); - prob = probs + IsMatch + (state << kNumPosBitsMax) + posState; + prob = probs + IsMatch + COMBINED_PS_STATE; IF_BIT_0_CHECK(prob) { UPDATE_0_CHECK @@ -618,10 +700,11 @@ static ELzmaDummy LzmaDec_TryDummy(const CLzmaDec *p, const Byte *buf, SizeT inS { unsigned bit; const CLzmaProb *probLit; - matchByte <<= 1; - bit = (matchByte & offs); - probLit = prob + offs + bit + symbol; - GET_BIT2_CHECK(probLit, symbol, offs &= ~bit, offs &= bit) + matchByte += matchByte; + bit = offs; + offs &= matchByte; + probLit = prob + (offs + bit + symbol); + GET_BIT2_CHECK(probLit, symbol, offs ^= bit; , ; ) } while (symbol < 0x100); } @@ -648,7 +731,7 @@ static ELzmaDummy LzmaDec_TryDummy(const CLzmaDec *p, const Byte *buf, SizeT inS IF_BIT_0_CHECK(prob) { UPDATE_0_CHECK; - prob = probs + IsRep0Long + (state << kNumPosBitsMax) + posState; + prob = probs + IsRep0Long + COMBINED_PS_STATE; IF_BIT_0_CHECK(prob) { UPDATE_0_CHECK; @@ -691,7 +774,7 @@ static ELzmaDummy LzmaDec_TryDummy(const CLzmaDec *p, const Byte *buf, SizeT inS IF_BIT_0_CHECK(probLen) { UPDATE_0_CHECK; - probLen = prob + LenLow + (posState << kLenNumLowBits); + probLen = prob + LenLow + GET_LEN_STATE; offset = 0; limit = 1 << kLenNumLowBits; } @@ -702,15 +785,15 @@ static ELzmaDummy LzmaDec_TryDummy(const CLzmaDec *p, const Byte *buf, SizeT inS IF_BIT_0_CHECK(probLen) { UPDATE_0_CHECK; - probLen = prob + LenMid + (posState << kLenNumMidBits); + probLen = prob + LenLow + GET_LEN_STATE + (1 << kLenNumLowBits); offset = kLenNumLowSymbols; - limit = 1 << kLenNumMidBits; + limit = 1 << kLenNumLowBits; } else { UPDATE_1_CHECK; probLen = prob + LenHigh; - offset = kLenNumLowSymbols + kLenNumMidSymbols; + offset = kLenNumLowSymbols * 2; limit = 1 << kLenNumHighBits; } } @@ -722,7 +805,7 @@ static ELzmaDummy LzmaDec_TryDummy(const CLzmaDec *p, const Byte *buf, SizeT inS { unsigned posSlot; prob = probs + PosSlot + - ((len < kNumLenToPosStates ? len : kNumLenToPosStates - 1) << + ((len < kNumLenToPosStates - 1 ? len : kNumLenToPosStates - 1) << kNumPosSlotBits); TREE_DECODE_CHECK(prob, 1 << kNumPosSlotBits, posSlot); if (posSlot >= kStartPosModelIndex) @@ -733,7 +816,7 @@ static ELzmaDummy LzmaDec_TryDummy(const CLzmaDec *p, const Byte *buf, SizeT inS if (posSlot < kEndPosModelIndex) { - prob = probs + SpecPos + ((2 | (posSlot & 1)) << numDirectBits) - posSlot - 1; + prob = probs + SpecPos + ((2 | (posSlot & 1)) << numDirectBits); } else { @@ -745,17 +828,18 @@ static ELzmaDummy LzmaDec_TryDummy(const CLzmaDec *p, const Byte *buf, SizeT inS code -= range & (((code - range) >> 31) - 1); /* if (code >= range) code -= range; */ } - while (--numDirectBits != 0); + while (--numDirectBits); prob = probs + Align; numDirectBits = kNumAlignBits; } { unsigned i = 1; + unsigned m = 1; do { - GET_BIT_CHECK(prob + i, i); + REV_BIT_CHECK(prob, i, m); } - while (--numDirectBits != 0); + while (--numDirectBits); } } } @@ -768,18 +852,17 @@ static ELzmaDummy LzmaDec_TryDummy(const CLzmaDec *p, const Byte *buf, SizeT inS void LzmaDec_InitDicAndState(CLzmaDec *p, Bool initDic, Bool initState) { - p->needFlush = 1; - p->remainLen = 0; + p->remainLen = kMatchSpecLenStart + 1; p->tempBufSize = 0; if (initDic) { p->processedPos = 0; p->checkDicSize = 0; - p->needInitState = 1; + p->remainLen = kMatchSpecLenStart + 2; } if (initState) - p->needInitState = 1; + p->remainLen = kMatchSpecLenStart + 2; } void LzmaDec_Init(CLzmaDec *p) @@ -788,53 +871,54 @@ void LzmaDec_Init(CLzmaDec *p) LzmaDec_InitDicAndState(p, True, True); } -static void LzmaDec_InitStateReal(CLzmaDec *p) -{ - SizeT numProbs = LzmaProps_GetNumProbs(&p->prop); - SizeT i; - CLzmaProb *probs = p->probs; - for (i = 0; i < numProbs; i++) - probs[i] = kBitModelTotal >> 1; - p->reps[0] = p->reps[1] = p->reps[2] = p->reps[3] = 1; - p->state = 0; - p->needInitState = 0; -} SRes LzmaDec_DecodeToDic(CLzmaDec *p, SizeT dicLimit, const Byte *src, SizeT *srcLen, ELzmaFinishMode finishMode, ELzmaStatus *status) { SizeT inSize = *srcLen; (*srcLen) = 0; - LzmaDec_WriteRem(p, dicLimit); *status = LZMA_STATUS_NOT_SPECIFIED; + if (p->remainLen > kMatchSpecLenStart) + { + for (; inSize > 0 && p->tempBufSize < RC_INIT_SIZE; (*srcLen)++, inSize--) + p->tempBuf[p->tempBufSize++] = *src++; + if (p->tempBufSize != 0 && p->tempBuf[0] != 0) + return SZ_ERROR_DATA; + if (p->tempBufSize < RC_INIT_SIZE) + { + *status = LZMA_STATUS_NEEDS_MORE_INPUT; + return SZ_OK; + } + p->code = + ((UInt32)p->tempBuf[1] << 24) + | ((UInt32)p->tempBuf[2] << 16) + | ((UInt32)p->tempBuf[3] << 8) + | ((UInt32)p->tempBuf[4]); + p->range = 0xFFFFFFFF; + p->tempBufSize = 0; + + if (p->remainLen > kMatchSpecLenStart + 1) + { + SizeT numProbs = LzmaProps_GetNumProbs(&p->prop); + SizeT i; + CLzmaProb *probs = p->probs; + for (i = 0; i < numProbs; i++) + probs[i] = kBitModelTotal >> 1; + p->reps[0] = p->reps[1] = p->reps[2] = p->reps[3] = 1; + p->state = 0; + } + + p->remainLen = 0; + } + + LzmaDec_WriteRem(p, dicLimit); + while (p->remainLen != kMatchSpecLenStart) { - int checkEndMarkNow; + int checkEndMarkNow = 0; - if (p->needFlush) - { - for (; inSize > 0 && p->tempBufSize < RC_INIT_SIZE; (*srcLen)++, inSize--) - p->tempBuf[p->tempBufSize++] = *src++; - if (p->tempBufSize < RC_INIT_SIZE) - { - *status = LZMA_STATUS_NEEDS_MORE_INPUT; - return SZ_OK; - } - if (p->tempBuf[0] != 0) - return SZ_ERROR_DATA; - p->code = - ((UInt32)p->tempBuf[1] << 24) - | ((UInt32)p->tempBuf[2] << 16) - | ((UInt32)p->tempBuf[3] << 8) - | ((UInt32)p->tempBuf[4]); - p->range = 0xFFFFFFFF; - p->needFlush = 0; - p->tempBufSize = 0; - } - - checkEndMarkNow = 0; if (p->dicPos >= dicLimit) { if (p->remainLen == 0 && p->code == 0) @@ -855,9 +939,6 @@ SRes LzmaDec_DecodeToDic(CLzmaDec *p, SizeT dicLimit, const Byte *src, SizeT *sr checkEndMarkNow = 1; } - if (p->needInitState) - LzmaDec_InitStateReal(p); - if (p->tempBufSize == 0) { SizeT processed; @@ -930,11 +1011,14 @@ SRes LzmaDec_DecodeToDic(CLzmaDec *p, SizeT dicLimit, const Byte *src, SizeT *sr p->tempBufSize = 0; } } - if (p->code == 0) - *status = LZMA_STATUS_FINISHED_WITH_MARK; - return (p->code == 0) ? SZ_OK : SZ_ERROR_DATA; + + if (p->code != 0) + return SZ_ERROR_DATA; + *status = LZMA_STATUS_FINISHED_WITH_MARK; + return SZ_OK; } + SRes LzmaDec_DecodeToBuf(CLzmaDec *p, Byte *dest, SizeT *destLen, const Byte *src, SizeT *srcLen, ELzmaFinishMode finishMode, ELzmaStatus *status) { SizeT outSize = *destLen; @@ -1011,10 +1095,10 @@ SRes LzmaProps_Decode(CLzmaProps *p, const Byte *data, unsigned size) if (d >= (9 * 5 * 5)) return SZ_ERROR_UNSUPPORTED; - p->lc = d % 9; + p->lc = (Byte)(d % 9); d /= 9; - p->pb = d / 5; - p->lp = d % 5; + p->pb = (Byte)(d / 5); + p->lp = (Byte)(d % 5); return SZ_OK; } @@ -1026,9 +1110,10 @@ static SRes LzmaDec_AllocateProbs2(CLzmaDec *p, const CLzmaProps *propNew, ISzAl { LzmaDec_FreeProbs(p, alloc); p->probs = (CLzmaProb *)ISzAlloc_Alloc(alloc, numProbs * sizeof(CLzmaProb)); - p->numProbs = numProbs; if (!p->probs) return SZ_ERROR_MEM; + p->probs_1664 = p->probs + 1664; + p->numProbs = numProbs; } return SZ_OK; } diff --git a/C/LzmaDec.h b/C/LzmaDec.h index a33bfbd0..28ce60c3 100644 --- a/C/LzmaDec.h +++ b/C/LzmaDec.h @@ -1,5 +1,5 @@ /* LzmaDec.h -- LZMA Decoder -2017-04-03 : Igor Pavlov : Public domain */ +2018-04-21 : Igor Pavlov : Public domain */ #ifndef __LZMA_DEC_H #define __LZMA_DEC_H @@ -12,11 +12,13 @@ EXTERN_C_BEGIN /* _LZMA_PROB32 can increase the speed on some CPUs, but memory usage for CLzmaDec::probs will be doubled in that case */ +typedef #ifdef _LZMA_PROB32 -#define CLzmaProb UInt32 + UInt32 #else -#define CLzmaProb UInt16 + UInt16 #endif + CLzmaProb; /* ---------- LZMA Properties ---------- */ @@ -25,7 +27,10 @@ EXTERN_C_BEGIN typedef struct _CLzmaProps { - unsigned lc, lp, pb; + Byte lc; + Byte lp; + Byte pb; + Byte _pad_; UInt32 dicSize; } CLzmaProps; @@ -47,32 +52,34 @@ SRes LzmaProps_Decode(CLzmaProps *p, const Byte *data, unsigned size); typedef struct { + /* Don't change this structure. ASM code can use it. */ CLzmaProps prop; CLzmaProb *probs; + CLzmaProb *probs_1664; Byte *dic; - const Byte *buf; - UInt32 range, code; - SizeT dicPos; SizeT dicBufSize; + SizeT dicPos; + const Byte *buf; + UInt32 range; + UInt32 code; UInt32 processedPos; UInt32 checkDicSize; - unsigned state; UInt32 reps[4]; - unsigned remainLen; - int needFlush; - int needInitState; + UInt32 state; + UInt32 remainLen; + UInt32 numProbs; unsigned tempBufSize; Byte tempBuf[LZMA_REQUIRED_INPUT_MAX]; } CLzmaDec; -#define LzmaDec_Construct(p) { (p)->dic = 0; (p)->probs = 0; } +#define LzmaDec_Construct(p) { (p)->dic = NULL; (p)->probs = NULL; } void LzmaDec_Init(CLzmaDec *p); /* There are two types of LZMA streams: - 0) Stream with end mark. That end mark adds about 6 bytes to compressed size. - 1) Stream without end mark. You must know exact uncompressed size to decompress such stream. */ + - Stream with end mark. That end mark adds about 6 bytes to compressed size. + - Stream without end mark. You must know exact uncompressed size to decompress such stream. */ typedef enum { @@ -132,8 +139,8 @@ LzmaDec_Allocate* can return: SRes LzmaDec_AllocateProbs(CLzmaDec *p, const Byte *props, unsigned propsSize, ISzAllocPtr alloc); void LzmaDec_FreeProbs(CLzmaDec *p, ISzAllocPtr alloc); -SRes LzmaDec_Allocate(CLzmaDec *state, const Byte *prop, unsigned propsSize, ISzAllocPtr alloc); -void LzmaDec_Free(CLzmaDec *state, ISzAllocPtr alloc); +SRes LzmaDec_Allocate(CLzmaDec *p, const Byte *props, unsigned propsSize, ISzAllocPtr alloc); +void LzmaDec_Free(CLzmaDec *p, ISzAllocPtr alloc); /* ---------- Dictionary Interface ---------- */ @@ -142,7 +149,7 @@ void LzmaDec_Free(CLzmaDec *state, ISzAllocPtr alloc); You must work with CLzmaDec variables directly in this interface. STEPS: - LzmaDec_Constr() + LzmaDec_Construct() LzmaDec_Allocate() for (each new stream) { diff --git a/C/LzmaEnc.c b/C/LzmaEnc.c index 9c45a214..bebe664d 100644 --- a/C/LzmaEnc.c +++ b/C/LzmaEnc.c @@ -1,5 +1,5 @@ /* LzmaEnc.c -- LZMA Encoder -2017-06-22 : Igor Pavlov : Public domain */ +2018-04-29 : Igor Pavlov : Public domain */ #include "Precomp.h" @@ -26,15 +26,6 @@ static unsigned g_STAT_OFFSET = 0; #define kLzmaMaxHistorySize ((UInt32)3 << 29) /* #define kLzmaMaxHistorySize ((UInt32)7 << 29) */ -#define kBlockSizeMax ((1 << LZMA_NUM_BLOCK_SIZE_BITS) - 1) - -#define kBlockSize (9 << 10) -#define kUnpackBlockSize (1 << 18) -#define kMatchArraySize (1 << 21) -#define kMatchRecordMaxSize ((LZMA_MATCH_LEN_MAX * 2 + 3) * LZMA_MATCH_LEN_MAX) - -#define kNumMaxDirectBits (31) - #define kNumTopBits 24 #define kTopValue ((UInt32)1 << kNumTopBits) @@ -111,9 +102,9 @@ UInt32 LzmaEncProps_GetDictSize(const CLzmaEncProps *props2) #define BSR2_RET(pos, res) { unsigned long zz; _BitScanReverse(&zz, (pos)); res = (zz + zz) + ((pos >> (zz - 1)) & 1); } -static UInt32 GetPosSlot1(UInt32 pos) +static unsigned GetPosSlot1(UInt32 pos) { - UInt32 res; + unsigned res; BSR2_RET(pos, res); return res; } @@ -146,18 +137,18 @@ static void LzmaEnc_FastPosInit(Byte *g_FastPos) /* we can use ((limit - pos) >> 31) only if (pos < ((UInt32)1 << 31)) */ /* -#define BSR2_RET(pos, res) { UInt32 zz = 6 + ((kNumLogBits - 1) & \ +#define BSR2_RET(pos, res) { unsigned zz = 6 + ((kNumLogBits - 1) & \ (0 - (((((UInt32)1 << (kNumLogBits + 6)) - 1) - pos) >> 31))); \ res = p->g_FastPos[pos >> zz] + (zz * 2); } */ /* -#define BSR2_RET(pos, res) { UInt32 zz = 6 + ((kNumLogBits - 1) & \ +#define BSR2_RET(pos, res) { unsigned zz = 6 + ((kNumLogBits - 1) & \ (0 - (((((UInt32)1 << (kNumLogBits)) - 1) - (pos >> 6)) >> 31))); \ res = p->g_FastPos[pos >> zz] + (zz * 2); } */ -#define BSR2_RET(pos, res) { UInt32 zz = (pos < (1 << (kNumLogBits + 6))) ? 6 : 6 + kNumLogBits - 1; \ +#define BSR2_RET(pos, res) { unsigned zz = (pos < (1 << (kNumLogBits + 6))) ? 6 : 6 + kNumLogBits - 1; \ res = p->g_FastPos[pos >> zz] + (zz * 2); } /* @@ -168,32 +159,32 @@ static void LzmaEnc_FastPosInit(Byte *g_FastPos) #define GetPosSlot1(pos) p->g_FastPos[pos] #define GetPosSlot2(pos, res) { BSR2_RET(pos, res); } -#define GetPosSlot(pos, res) { if (pos < kNumFullDistances) res = p->g_FastPos[pos]; else BSR2_RET(pos, res); } +#define GetPosSlot(pos, res) { if (pos < kNumFullDistances) res = p->g_FastPos[pos & (kNumFullDistances - 1)]; else BSR2_RET(pos, res); } #endif #define LZMA_NUM_REPS 4 -typedef unsigned CState; +typedef UInt16 CState; +typedef UInt16 CExtra; typedef struct { UInt32 price; - CState state; - int prev1IsChar; - int prev2; - - UInt32 posPrev2; - UInt32 backPrev2; - - UInt32 posPrev; - UInt32 backPrev; - UInt32 backs[LZMA_NUM_REPS]; + CExtra extra; + // 0 : normal + // 1 : LIT : MATCH + // > 1 : MATCH (extra-1) : LIT : REP0 (len) + UInt32 len; + UInt32 dist; + UInt32 reps[LZMA_NUM_REPS]; } COptimal; + #define kNumOpts (1 << 12) +#define kPackReserve (1 + kNumOpts * 2) #define kNumLenToPosStates 4 #define kNumPosSlotBits 6 @@ -201,22 +192,21 @@ typedef struct #define kDicLogSizeMax 32 #define kDistTableSizeMax (kDicLogSizeMax * 2) - #define kNumAlignBits 4 #define kAlignTableSize (1 << kNumAlignBits) #define kAlignMask (kAlignTableSize - 1) #define kStartPosModelIndex 4 #define kEndPosModelIndex 14 -#define kNumPosModels (kEndPosModelIndex - kStartPosModelIndex) - #define kNumFullDistances (1 << (kEndPosModelIndex >> 1)) +typedef #ifdef _LZMA_PROB32 -#define CLzmaProb UInt32 + UInt32 #else -#define CLzmaProb UInt16 + UInt16 #endif + CLzmaProb; #define LZMA_PB_MAX 4 #define LZMA_LC_MAX 8 @@ -224,15 +214,11 @@ typedef struct #define LZMA_NUM_PB_STATES_MAX (1 << LZMA_PB_MAX) - #define kLenNumLowBits 3 #define kLenNumLowSymbols (1 << kLenNumLowBits) -#define kLenNumMidBits 3 -#define kLenNumMidSymbols (1 << kLenNumMidBits) #define kLenNumHighBits 8 #define kLenNumHighSymbols (1 << kLenNumHighBits) - -#define kLenNumSymbolsTotal (kLenNumLowSymbols + kLenNumMidSymbols + kLenNumHighSymbols) +#define kLenNumSymbolsTotal (kLenNumLowSymbols * 2 + kLenNumHighSymbols) #define LZMA_MATCH_LEN_MIN 2 #define LZMA_MATCH_LEN_MAX (LZMA_MATCH_LEN_MIN + kLenNumSymbolsTotal - 1) @@ -242,27 +228,23 @@ typedef struct typedef struct { - CLzmaProb choice; - CLzmaProb choice2; - CLzmaProb low[LZMA_NUM_PB_STATES_MAX << kLenNumLowBits]; - CLzmaProb mid[LZMA_NUM_PB_STATES_MAX << kLenNumMidBits]; + CLzmaProb low[LZMA_NUM_PB_STATES_MAX << (kLenNumLowBits + 1)]; CLzmaProb high[kLenNumHighSymbols]; } CLenEnc; typedef struct { - CLenEnc p; - UInt32 tableSize; + unsigned tableSize; + unsigned counters[LZMA_NUM_PB_STATES_MAX]; UInt32 prices[LZMA_NUM_PB_STATES_MAX][kLenNumSymbolsTotal]; - UInt32 counters[LZMA_NUM_PB_STATES_MAX]; } CLenPriceEnc; typedef struct { UInt32 range; - Byte cache; + unsigned cache; UInt64 low; UInt64 cacheSize; Byte *buf; @@ -278,48 +260,54 @@ typedef struct { CLzmaProb *litProbs; - UInt32 state; + unsigned state; UInt32 reps[LZMA_NUM_REPS]; - CLzmaProb isMatch[kNumStates][LZMA_NUM_PB_STATES_MAX]; + CLzmaProb posAlignEncoder[1 << kNumAlignBits]; CLzmaProb isRep[kNumStates]; CLzmaProb isRepG0[kNumStates]; CLzmaProb isRepG1[kNumStates]; CLzmaProb isRepG2[kNumStates]; + CLzmaProb isMatch[kNumStates][LZMA_NUM_PB_STATES_MAX]; CLzmaProb isRep0Long[kNumStates][LZMA_NUM_PB_STATES_MAX]; CLzmaProb posSlotEncoder[kNumLenToPosStates][1 << kNumPosSlotBits]; - CLzmaProb posEncoders[kNumFullDistances - kEndPosModelIndex]; - CLzmaProb posAlignEncoder[1 << kNumAlignBits]; + CLzmaProb posEncoders[kNumFullDistances]; - CLenPriceEnc lenEnc; - CLenPriceEnc repLenEnc; + CLenEnc lenProbs; + CLenEnc repLenProbs; + } CSaveState; +typedef UInt32 CProbPrice; + + typedef struct { void *matchFinderObj; IMatchFinder matchFinder; - UInt32 optimumEndIndex; - UInt32 optimumCurrentIndex; + unsigned optCur; + unsigned optEnd; - UInt32 longestMatchLength; - UInt32 numPairs; + unsigned longestMatchLen; + unsigned numPairs; UInt32 numAvail; - UInt32 numFastBytes; - UInt32 additionalOffset; + unsigned state; + unsigned numFastBytes; + unsigned additionalOffset; UInt32 reps[LZMA_NUM_REPS]; - UInt32 state; + unsigned lpMask, pbMask; + CLzmaProb *litProbs; + CRangeEnc rc; + + UInt32 backRes; unsigned lc, lp, pb; - unsigned lpMask, pbMask; unsigned lclp; - CLzmaProb *litProbs; - Bool fastMode; Bool writeEndMark; Bool finished; @@ -328,19 +316,19 @@ typedef struct UInt64 nowPos64; - UInt32 matchPriceCount; - UInt32 alignPriceCount; + unsigned matchPriceCount; + unsigned alignPriceCount; - UInt32 distTableSize; + unsigned distTableSize; UInt32 dictSize; SRes result; - CRangeEnc rc; - #ifndef _7ZIP_ST Bool mtMode; + // begin of CMatchFinderMt is used in LZ thread CMatchFinderMt matchFinderMt; + // end of CMatchFinderMt is used in BT and HASH threads #endif CMatchFinder matchFinderBase; @@ -349,33 +337,37 @@ typedef struct Byte pad[128]; #endif - COptimal opt[kNumOpts]; - - #ifndef LZMA_LOG_BSR - Byte g_FastPos[1 << kNumLogBits]; - #endif + // LZ thread + CProbPrice ProbPrices[kBitModelTotal >> kNumMoveReducingBits]; - UInt32 ProbPrices[kBitModelTotal >> kNumMoveReducingBits]; UInt32 matches[LZMA_MATCH_LEN_MAX * 2 + 2 + 1]; + UInt32 alignPrices[kAlignTableSize]; UInt32 posSlotPrices[kNumLenToPosStates][kDistTableSizeMax]; UInt32 distancesPrices[kNumLenToPosStates][kNumFullDistances]; - UInt32 alignPrices[kAlignTableSize]; - CLzmaProb isMatch[kNumStates][LZMA_NUM_PB_STATES_MAX]; + CLzmaProb posAlignEncoder[1 << kNumAlignBits]; CLzmaProb isRep[kNumStates]; CLzmaProb isRepG0[kNumStates]; CLzmaProb isRepG1[kNumStates]; CLzmaProb isRepG2[kNumStates]; + CLzmaProb isMatch[kNumStates][LZMA_NUM_PB_STATES_MAX]; CLzmaProb isRep0Long[kNumStates][LZMA_NUM_PB_STATES_MAX]; - CLzmaProb posSlotEncoder[kNumLenToPosStates][1 << kNumPosSlotBits]; - CLzmaProb posEncoders[kNumFullDistances - kEndPosModelIndex]; - CLzmaProb posAlignEncoder[1 << kNumAlignBits]; + CLzmaProb posEncoders[kNumFullDistances]; + CLenEnc lenProbs; + CLenEnc repLenProbs; + + #ifndef LZMA_LOG_BSR + Byte g_FastPos[1 << kNumLogBits]; + #endif + CLenPriceEnc lenEnc; CLenPriceEnc repLenEnc; + COptimal opt[kNumOpts]; + CSaveState saveState; #ifndef _7ZIP_ST @@ -384,58 +376,62 @@ typedef struct } CLzmaEnc; + +#define COPY_ARR(dest, src, arr) memcpy(dest->arr, src->arr, sizeof(src->arr)); + void LzmaEnc_SaveState(CLzmaEncHandle pp) { CLzmaEnc *p = (CLzmaEnc *)pp; CSaveState *dest = &p->saveState; - int i; - dest->lenEnc = p->lenEnc; - dest->repLenEnc = p->repLenEnc; + dest->state = p->state; + + dest->lenProbs = p->lenProbs; + dest->repLenProbs = p->repLenProbs; + + COPY_ARR(dest, p, reps); + + COPY_ARR(dest, p, posAlignEncoder); + COPY_ARR(dest, p, isRep); + COPY_ARR(dest, p, isRepG0); + COPY_ARR(dest, p, isRepG1); + COPY_ARR(dest, p, isRepG2); + COPY_ARR(dest, p, isMatch); + COPY_ARR(dest, p, isRep0Long); + COPY_ARR(dest, p, posSlotEncoder); + COPY_ARR(dest, p, posEncoders); - for (i = 0; i < kNumStates; i++) - { - memcpy(dest->isMatch[i], p->isMatch[i], sizeof(p->isMatch[i])); - memcpy(dest->isRep0Long[i], p->isRep0Long[i], sizeof(p->isRep0Long[i])); - } - for (i = 0; i < kNumLenToPosStates; i++) - memcpy(dest->posSlotEncoder[i], p->posSlotEncoder[i], sizeof(p->posSlotEncoder[i])); - memcpy(dest->isRep, p->isRep, sizeof(p->isRep)); - memcpy(dest->isRepG0, p->isRepG0, sizeof(p->isRepG0)); - memcpy(dest->isRepG1, p->isRepG1, sizeof(p->isRepG1)); - memcpy(dest->isRepG2, p->isRepG2, sizeof(p->isRepG2)); - memcpy(dest->posEncoders, p->posEncoders, sizeof(p->posEncoders)); - memcpy(dest->posAlignEncoder, p->posAlignEncoder, sizeof(p->posAlignEncoder)); - memcpy(dest->reps, p->reps, sizeof(p->reps)); memcpy(dest->litProbs, p->litProbs, ((UInt32)0x300 << p->lclp) * sizeof(CLzmaProb)); } + void LzmaEnc_RestoreState(CLzmaEncHandle pp) { CLzmaEnc *dest = (CLzmaEnc *)pp; const CSaveState *p = &dest->saveState; - int i; - dest->lenEnc = p->lenEnc; - dest->repLenEnc = p->repLenEnc; + dest->state = p->state; - for (i = 0; i < kNumStates; i++) - { - memcpy(dest->isMatch[i], p->isMatch[i], sizeof(p->isMatch[i])); - memcpy(dest->isRep0Long[i], p->isRep0Long[i], sizeof(p->isRep0Long[i])); - } - for (i = 0; i < kNumLenToPosStates; i++) - memcpy(dest->posSlotEncoder[i], p->posSlotEncoder[i], sizeof(p->posSlotEncoder[i])); - memcpy(dest->isRep, p->isRep, sizeof(p->isRep)); - memcpy(dest->isRepG0, p->isRepG0, sizeof(p->isRepG0)); - memcpy(dest->isRepG1, p->isRepG1, sizeof(p->isRepG1)); - memcpy(dest->isRepG2, p->isRepG2, sizeof(p->isRepG2)); - memcpy(dest->posEncoders, p->posEncoders, sizeof(p->posEncoders)); - memcpy(dest->posAlignEncoder, p->posAlignEncoder, sizeof(p->posAlignEncoder)); - memcpy(dest->reps, p->reps, sizeof(p->reps)); + dest->lenProbs = p->lenProbs; + dest->repLenProbs = p->repLenProbs; + + COPY_ARR(dest, p, reps); + + COPY_ARR(dest, p, posAlignEncoder); + COPY_ARR(dest, p, isRep); + COPY_ARR(dest, p, isRepG0); + COPY_ARR(dest, p, isRepG1); + COPY_ARR(dest, p, isRepG2); + COPY_ARR(dest, p, isMatch); + COPY_ARR(dest, p, isRep0Long); + COPY_ARR(dest, p, posSlotEncoder); + COPY_ARR(dest, p, posEncoders); + memcpy(dest->litProbs, p->litProbs, ((UInt32)0x300 << dest->lclp) * sizeof(CLzmaProb)); } + + SRes LzmaEnc_SetProps(CLzmaEncHandle pp, const CLzmaEncProps *props2) { CLzmaEnc *p = (CLzmaEnc *)pp; @@ -464,7 +460,7 @@ SRes LzmaEnc_SetProps(CLzmaEncHandle pp, const CLzmaEncProps *props2) p->fastMode = (props.algo == 0); p->matchFinderBase.btMode = (Byte)(props.btMode ? 1 : 0); { - UInt32 numHashBytes = 4; + unsigned numHashBytes = 4; if (props.btMode) { if (props.numHashBytes < 2) @@ -501,14 +497,19 @@ void LzmaEnc_SetDataSize(CLzmaEncHandle pp, UInt64 expectedDataSiize) } +#define kState_Start 0 +#define kState_LitAfterMatch 4 +#define kState_LitAfterRep 5 +#define kState_MatchAfterLit 7 +#define kState_RepAfterLit 8 -static const int kLiteralNextStates[kNumStates] = {0, 0, 0, 0, 1, 2, 3, 4, 5, 6, 4, 5}; -static const int kMatchNextStates[kNumStates] = {7, 7, 7, 7, 7, 7, 7, 10, 10, 10, 10, 10}; -static const int kRepNextStates[kNumStates] = {8, 8, 8, 8, 8, 8, 8, 11, 11, 11, 11, 11}; -static const int kShortRepNextStates[kNumStates]= {9, 9, 9, 9, 9, 9, 9, 11, 11, 11, 11, 11}; - -#define IsCharState(s) ((s) < 7) +static const Byte kLiteralNextStates[kNumStates] = {0, 0, 0, 0, 1, 2, 3, 4, 5, 6, 4, 5}; +static const Byte kMatchNextStates[kNumStates] = {7, 7, 7, 7, 7, 7, 7, 10, 10, 10, 10, 10}; +static const Byte kRepNextStates[kNumStates] = {8, 8, 8, 8, 8, 8, 8, 11, 11, 11, 11, 11}; +static const Byte kShortRepNextStates[kNumStates]= {9, 9, 9, 9, 9, 9, 9, 11, 11, 11, 11, 11}; +#define IsLitState(s) ((s) < 7) +#define GetLenToPosState2(len) (((len) < kNumLenToPosStates - 1) ? (len) : kNumLenToPosStates - 1) #define GetLenToPosState(len) (((len) < kNumLenToPosStates + 1) ? (len) - 2 : kNumLenToPosStates - 1) #define kInfinityPrice (1 << 30) @@ -519,9 +520,11 @@ static void RangeEnc_Construct(CRangeEnc *p) p->bufBase = NULL; } -#define RangeEnc_GetProcessed(p) ((p)->processed + ((p)->buf - (p)->bufBase) + (p)->cacheSize) +#define RangeEnc_GetProcessed(p) ((p)->processed + ((p)->buf - (p)->bufBase) + (p)->cacheSize) +#define RangeEnc_GetProcessed_sizet(p) ((size_t)(p)->processed + ((p)->buf - (p)->bufBase) + (size_t)(p)->cacheSize) #define RC_BUF_SIZE (1 << 16) + static int RangeEnc_Alloc(CRangeEnc *p, ISzAllocPtr alloc) { if (!p->bufBase) @@ -543,10 +546,10 @@ static void RangeEnc_Free(CRangeEnc *p, ISzAllocPtr alloc) static void RangeEnc_Init(CRangeEnc *p) { /* Stream.Init(); */ - p->low = 0; p->range = 0xFFFFFFFF; - p->cacheSize = 1; p->cache = 0; + p->low = 0; + p->cacheSize = 0; p->buf = p->bufBase; @@ -554,7 +557,7 @@ static void RangeEnc_Init(CRangeEnc *p) p->res = SZ_OK; } -static void RangeEnc_FlushStream(CRangeEnc *p) +MY_NO_INLINE static void RangeEnc_FlushStream(CRangeEnc *p) { size_t num; if (p->res != SZ_OK) @@ -566,25 +569,36 @@ static void RangeEnc_FlushStream(CRangeEnc *p) p->buf = p->bufBase; } -static void MY_FAST_CALL RangeEnc_ShiftLow(CRangeEnc *p) +MY_NO_INLINE static void MY_FAST_CALL RangeEnc_ShiftLow(CRangeEnc *p) { - if ((UInt32)p->low < (UInt32)0xFF000000 || (unsigned)(p->low >> 32) != 0) + UInt32 low = (UInt32)p->low; + unsigned high = (unsigned)(p->low >> 32); + p->low = (UInt32)(low << 8); + if (low < (UInt32)0xFF000000 || high != 0) { - Byte temp = p->cache; - do { Byte *buf = p->buf; - *buf++ = (Byte)(temp + (Byte)(p->low >> 32)); + *buf++ = (Byte)(p->cache + high); + p->cache = (unsigned)(low >> 24); p->buf = buf; if (buf == p->bufLim) RangeEnc_FlushStream(p); - temp = 0xFF; + if (p->cacheSize == 0) + return; + } + high += 0xFF; + for (;;) + { + Byte *buf = p->buf; + *buf++ = (Byte)(high); + p->buf = buf; + if (buf == p->bufLim) + RangeEnc_FlushStream(p); + if (--p->cacheSize == 0) + return; } - while (--p->cacheSize != 0); - p->cache = (Byte)((UInt32)p->low >> 24); } p->cacheSize++; - p->low = (UInt32)p->low << 8; } static void RangeEnc_FlushData(CRangeEnc *p) @@ -594,78 +608,121 @@ static void RangeEnc_FlushData(CRangeEnc *p) RangeEnc_ShiftLow(p); } -static void RangeEnc_EncodeDirectBits(CRangeEnc *p, UInt32 value, unsigned numBits) -{ - do - { - p->range >>= 1; - p->low += p->range & (0 - ((value >> --numBits) & 1)); - if (p->range < kTopValue) - { - p->range <<= 8; - RangeEnc_ShiftLow(p); - } - } - while (numBits != 0); -} +#define RC_NORM(p) if (range < kTopValue) { range <<= 8; RangeEnc_ShiftLow(p); } -static void RangeEnc_EncodeBit(CRangeEnc *p, CLzmaProb *prob, UInt32 symbol) +#define RC_BIT_PRE(p, prob) \ + ttt = *(prob); \ + newBound = (range >> kNumBitModelTotalBits) * ttt; + +// #define _LZMA_ENC_USE_BRANCH + +#ifdef _LZMA_ENC_USE_BRANCH + +#define RC_BIT(p, prob, symbol) { \ + RC_BIT_PRE(p, prob) \ + if (symbol == 0) { range = newBound; ttt += (kBitModelTotal - ttt) >> kNumMoveBits; } \ + else { (p)->low += newBound; range -= newBound; ttt -= ttt >> kNumMoveBits; } \ + *(prob) = (CLzmaProb)ttt; \ + RC_NORM(p) \ + } + +#else + +#define RC_BIT(p, prob, symbol) { \ + UInt32 mask; \ + RC_BIT_PRE(p, prob) \ + mask = 0 - (UInt32)symbol; \ + range &= mask; \ + mask &= newBound; \ + range -= mask; \ + (p)->low += mask; \ + mask = (UInt32)symbol - 1; \ + range += newBound & mask; \ + mask &= (kBitModelTotal - ((1 << kNumMoveBits) - 1)); \ + mask += ((1 << kNumMoveBits) - 1); \ + ttt += (Int32)(mask - ttt) >> kNumMoveBits; \ + *(prob) = (CLzmaProb)ttt; \ + RC_NORM(p) \ + } + +#endif + + + + +#define RC_BIT_0_BASE(p, prob) \ + range = newBound; *(prob) = (CLzmaProb)(ttt + ((kBitModelTotal - ttt) >> kNumMoveBits)); + +#define RC_BIT_1_BASE(p, prob) \ + range -= newBound; (p)->low += newBound; *(prob) = (CLzmaProb)(ttt - (ttt >> kNumMoveBits)); \ + +#define RC_BIT_0(p, prob) \ + RC_BIT_0_BASE(p, prob) \ + RC_NORM(p) + +#define RC_BIT_1(p, prob) \ + RC_BIT_1_BASE(p, prob) \ + RC_NORM(p) + +static void RangeEnc_EncodeBit_0(CRangeEnc *p, CLzmaProb *prob) { - UInt32 ttt = *prob; - UInt32 newBound = (p->range >> kNumBitModelTotalBits) * ttt; - if (symbol == 0) - { - p->range = newBound; - ttt += (kBitModelTotal - ttt) >> kNumMoveBits; - } - else - { - p->low += newBound; - p->range -= newBound; - ttt -= ttt >> kNumMoveBits; - } - *prob = (CLzmaProb)ttt; - if (p->range < kTopValue) - { - p->range <<= 8; - RangeEnc_ShiftLow(p); - } + UInt32 range, ttt, newBound; + range = p->range; + RC_BIT_PRE(p, prob) + RC_BIT_0(p, prob) + p->range = range; } static void LitEnc_Encode(CRangeEnc *p, CLzmaProb *probs, UInt32 symbol) { + UInt32 range = p->range; symbol |= 0x100; do { - RangeEnc_EncodeBit(p, probs + (symbol >> 8), (symbol >> 7) & 1); + UInt32 ttt, newBound; + // RangeEnc_EncodeBit(p, probs + (symbol >> 8), (symbol >> 7) & 1); + CLzmaProb *prob = probs + (symbol >> 8); + UInt32 bit = (symbol >> 7) & 1; symbol <<= 1; + RC_BIT(p, prob, bit); } while (symbol < 0x10000); + p->range = range; } static void LitEnc_EncodeMatched(CRangeEnc *p, CLzmaProb *probs, UInt32 symbol, UInt32 matchByte) { + UInt32 range = p->range; UInt32 offs = 0x100; symbol |= 0x100; do { + UInt32 ttt, newBound; + CLzmaProb *prob; + UInt32 bit; matchByte <<= 1; - RangeEnc_EncodeBit(p, probs + (offs + (matchByte & offs) + (symbol >> 8)), (symbol >> 7) & 1); + // RangeEnc_EncodeBit(p, probs + (offs + (matchByte & offs) + (symbol >> 8)), (symbol >> 7) & 1); + prob = probs + (offs + (matchByte & offs) + (symbol >> 8)); + bit = (symbol >> 7) & 1; symbol <<= 1; offs &= ~(matchByte ^ symbol); + RC_BIT(p, prob, bit); } while (symbol < 0x10000); + p->range = range; } -static void LzmaEnc_InitPriceTables(UInt32 *ProbPrices) + + +static void LzmaEnc_InitPriceTables(CProbPrice *ProbPrices) { UInt32 i; - for (i = (1 << kNumMoveReducingBits) / 2; i < kBitModelTotal; i += (1 << kNumMoveReducingBits)) + for (i = 0; i < (kBitModelTotal >> kNumMoveReducingBits); i++) { - const int kCyclesBits = kNumBitPriceShiftBits; - UInt32 w = i; - UInt32 bitCount = 0; - int j; + const unsigned kCyclesBits = kNumBitPriceShiftBits; + UInt32 w = (i << kNumMoveReducingBits) + (1 << (kNumMoveReducingBits - 1)); + unsigned bitCount = 0; + unsigned j; for (j = 0; j < kCyclesBits; j++) { w = w * w; @@ -676,37 +733,41 @@ static void LzmaEnc_InitPriceTables(UInt32 *ProbPrices) bitCount++; } } - ProbPrices[i >> kNumMoveReducingBits] = ((kNumBitModelTotalBits << kCyclesBits) - 15 - bitCount); + ProbPrices[i] = (CProbPrice)((kNumBitModelTotalBits << kCyclesBits) - 15 - bitCount); + // printf("\n%3d: %5d", i, ProbPrices[i]); } } #define GET_PRICE(prob, symbol) \ - p->ProbPrices[((prob) ^ (((-(int)(symbol))) & (kBitModelTotal - 1))) >> kNumMoveReducingBits]; + p->ProbPrices[((prob) ^ (unsigned)(((-(int)(symbol))) & (kBitModelTotal - 1))) >> kNumMoveReducingBits]; #define GET_PRICEa(prob, symbol) \ - ProbPrices[((prob) ^ ((-((int)(symbol))) & (kBitModelTotal - 1))) >> kNumMoveReducingBits]; + ProbPrices[((prob) ^ (unsigned)((-((int)(symbol))) & (kBitModelTotal - 1))) >> kNumMoveReducingBits]; #define GET_PRICE_0(prob) p->ProbPrices[(prob) >> kNumMoveReducingBits] #define GET_PRICE_1(prob) p->ProbPrices[((prob) ^ (kBitModelTotal - 1)) >> kNumMoveReducingBits] -#define GET_PRICE_0a(prob) ProbPrices[(prob) >> kNumMoveReducingBits] -#define GET_PRICE_1a(prob) ProbPrices[((prob) ^ (kBitModelTotal - 1)) >> kNumMoveReducingBits] +#define GET_PRICEa_0(prob) ProbPrices[(prob) >> kNumMoveReducingBits] +#define GET_PRICEa_1(prob) ProbPrices[((prob) ^ (kBitModelTotal - 1)) >> kNumMoveReducingBits] -static UInt32 LitEnc_GetPrice(const CLzmaProb *probs, UInt32 symbol, const UInt32 *ProbPrices) + +static UInt32 LitEnc_GetPrice(const CLzmaProb *probs, UInt32 symbol, const CProbPrice *ProbPrices) { UInt32 price = 0; symbol |= 0x100; do { - price += GET_PRICEa(probs[symbol >> 8], (symbol >> 7) & 1); - symbol <<= 1; + unsigned bit = symbol & 1; + symbol >>= 1; + price += GET_PRICEa(probs[symbol], bit); } - while (symbol < 0x10000); + while (symbol >= 2); return price; } -static UInt32 LitEnc_GetPriceMatched(const CLzmaProb *probs, UInt32 symbol, UInt32 matchByte, const UInt32 *ProbPrices) + +static UInt32 LitEnc_Matched_GetPrice(const CLzmaProb *probs, UInt32 symbol, UInt32 matchByte, const CProbPrice *ProbPrices) { UInt32 price = 0; UInt32 offs = 0x100; @@ -723,520 +784,525 @@ static UInt32 LitEnc_GetPriceMatched(const CLzmaProb *probs, UInt32 symbol, UInt } -static void RcTree_Encode(CRangeEnc *rc, CLzmaProb *probs, int numBitLevels, UInt32 symbol) +static void RcTree_ReverseEncode(CRangeEnc *rc, CLzmaProb *probs, unsigned numBits, UInt32 symbol) { - UInt32 m = 1; - int i; - for (i = numBitLevels; i != 0;) + UInt32 range = rc->range; + unsigned m = 1; + do { - UInt32 bit; - i--; - bit = (symbol >> i) & 1; - RangeEnc_EncodeBit(rc, probs + m, bit); + UInt32 ttt, newBound; + unsigned bit = symbol & 1; + // RangeEnc_EncodeBit(rc, probs + m, bit); + symbol >>= 1; + RC_BIT(rc, probs + m, bit); m = (m << 1) | bit; } + while (--numBits); + rc->range = range; } -static void RcTree_ReverseEncode(CRangeEnc *rc, CLzmaProb *probs, int numBitLevels, UInt32 symbol) -{ - UInt32 m = 1; - int i; - for (i = 0; i < numBitLevels; i++) - { - UInt32 bit = symbol & 1; - RangeEnc_EncodeBit(rc, probs + m, bit); - m = (m << 1) | bit; - symbol >>= 1; - } -} - -static UInt32 RcTree_GetPrice(const CLzmaProb *probs, int numBitLevels, UInt32 symbol, const UInt32 *ProbPrices) -{ - UInt32 price = 0; - symbol |= (1 << numBitLevels); - while (symbol != 1) - { - price += GET_PRICEa(probs[symbol >> 1], symbol & 1); - symbol >>= 1; - } - return price; -} - -static UInt32 RcTree_ReverseGetPrice(const CLzmaProb *probs, int numBitLevels, UInt32 symbol, const UInt32 *ProbPrices) -{ - UInt32 price = 0; - UInt32 m = 1; - int i; - for (i = numBitLevels; i != 0; i--) - { - UInt32 bit = symbol & 1; - symbol >>= 1; - price += GET_PRICEa(probs[m], bit); - m = (m << 1) | bit; - } - return price; -} static void LenEnc_Init(CLenEnc *p) { unsigned i; - p->choice = p->choice2 = kProbInitValue; - for (i = 0; i < (LZMA_NUM_PB_STATES_MAX << kLenNumLowBits); i++) + for (i = 0; i < (LZMA_NUM_PB_STATES_MAX << (kLenNumLowBits + 1)); i++) p->low[i] = kProbInitValue; - for (i = 0; i < (LZMA_NUM_PB_STATES_MAX << kLenNumMidBits); i++) - p->mid[i] = kProbInitValue; for (i = 0; i < kLenNumHighSymbols; i++) p->high[i] = kProbInitValue; } -static void LenEnc_Encode(CLenEnc *p, CRangeEnc *rc, UInt32 symbol, UInt32 posState) +static void LenEnc_Encode(CLenEnc *p, CRangeEnc *rc, unsigned symbol, unsigned posState) { - if (symbol < kLenNumLowSymbols) + UInt32 range, ttt, newBound; + CLzmaProb *probs = p->low; + range = rc->range; + RC_BIT_PRE(rc, probs); + if (symbol >= kLenNumLowSymbols) { - RangeEnc_EncodeBit(rc, &p->choice, 0); - RcTree_Encode(rc, p->low + (posState << kLenNumLowBits), kLenNumLowBits, symbol); - } - else - { - RangeEnc_EncodeBit(rc, &p->choice, 1); - if (symbol < kLenNumLowSymbols + kLenNumMidSymbols) + RC_BIT_1(rc, probs); + probs += kLenNumLowSymbols; + RC_BIT_PRE(rc, probs); + if (symbol >= kLenNumLowSymbols * 2) { - RangeEnc_EncodeBit(rc, &p->choice2, 0); - RcTree_Encode(rc, p->mid + (posState << kLenNumMidBits), kLenNumMidBits, symbol - kLenNumLowSymbols); - } - else - { - RangeEnc_EncodeBit(rc, &p->choice2, 1); - RcTree_Encode(rc, p->high, kLenNumHighBits, symbol - kLenNumLowSymbols - kLenNumMidSymbols); - } - } -} - -static void LenEnc_SetPrices(CLenEnc *p, UInt32 posState, UInt32 numSymbols, UInt32 *prices, const UInt32 *ProbPrices) -{ - UInt32 a0 = GET_PRICE_0a(p->choice); - UInt32 a1 = GET_PRICE_1a(p->choice); - UInt32 b0 = a1 + GET_PRICE_0a(p->choice2); - UInt32 b1 = a1 + GET_PRICE_1a(p->choice2); - UInt32 i = 0; - for (i = 0; i < kLenNumLowSymbols; i++) - { - if (i >= numSymbols) + RC_BIT_1(rc, probs); + rc->range = range; + // RcTree_Encode(rc, p->high, kLenNumHighBits, symbol - kLenNumLowSymbols * 2); + LitEnc_Encode(rc, p->high, symbol - kLenNumLowSymbols * 2); return; - prices[i] = a0 + RcTree_GetPrice(p->low + (posState << kLenNumLowBits), kLenNumLowBits, i, ProbPrices); + } + symbol -= kLenNumLowSymbols; } - for (; i < kLenNumLowSymbols + kLenNumMidSymbols; i++) + + // RcTree_Encode(rc, probs + (posState << kLenNumLowBits), kLenNumLowBits, symbol); { - if (i >= numSymbols) - return; - prices[i] = b0 + RcTree_GetPrice(p->mid + (posState << kLenNumMidBits), kLenNumMidBits, i - kLenNumLowSymbols, ProbPrices); + unsigned m; + unsigned bit; + RC_BIT_0(rc, probs); + probs += (posState << (1 + kLenNumLowBits)); + bit = (symbol >> 2) ; RC_BIT(rc, probs + 1, bit); m = (1 << 1) + bit; + bit = (symbol >> 1) & 1; RC_BIT(rc, probs + m, bit); m = (m << 1) + bit; + bit = symbol & 1; RC_BIT(rc, probs + m, bit); + rc->range = range; } - for (; i < numSymbols; i++) - prices[i] = b1 + RcTree_GetPrice(p->high, kLenNumHighBits, i - kLenNumLowSymbols - kLenNumMidSymbols, ProbPrices); } -static void MY_FAST_CALL LenPriceEnc_UpdateTable(CLenPriceEnc *p, UInt32 posState, const UInt32 *ProbPrices) +static void SetPrices_3(const CLzmaProb *probs, UInt32 startPrice, UInt32 *prices, const CProbPrice *ProbPrices) { - LenEnc_SetPrices(&p->p, posState, p->tableSize, p->prices[posState], ProbPrices); - p->counters[posState] = p->tableSize; + unsigned i; + for (i = 0; i < 8; i += 2) + { + UInt32 price = startPrice; + UInt32 prob; + price += GET_PRICEa(probs[1 ], (i >> 2)); + price += GET_PRICEa(probs[2 + (i >> 2)], (i >> 1) & 1); + prob = probs[4 + (i >> 1)]; + prices[i ] = price + GET_PRICEa_0(prob); + prices[i + 1] = price + GET_PRICEa_1(prob); + } } -static void LenPriceEnc_UpdateTables(CLenPriceEnc *p, UInt32 numPosStates, const UInt32 *ProbPrices) + +MY_NO_INLINE static void MY_FAST_CALL LenPriceEnc_UpdateTable( + CLenPriceEnc *p, unsigned posState, + const CLenEnc *enc, + const CProbPrice *ProbPrices) { - UInt32 posState; + // int y; for (y = 0; y < 100; y++) { + UInt32 a; + unsigned i, numSymbols; + + UInt32 *prices = p->prices[posState]; + { + const CLzmaProb *probs = enc->low + (posState << (1 + kLenNumLowBits)); + SetPrices_3(probs, GET_PRICEa_0(enc->low[0]), prices, ProbPrices); + a = GET_PRICEa_1(enc->low[0]); + SetPrices_3(probs + kLenNumLowSymbols, a + GET_PRICEa_0(enc->low[kLenNumLowSymbols]), prices + kLenNumLowSymbols, ProbPrices); + a += GET_PRICEa_1(enc->low[kLenNumLowSymbols]); + } + numSymbols = p->tableSize; + p->counters[posState] = numSymbols; + for (i = kLenNumLowSymbols * 2; i < numSymbols; i += 1) + { + prices[i] = a + + // RcTree_GetPrice(enc->high, kLenNumHighBits, i - kLenNumLowSymbols * 2, ProbPrices); + LitEnc_GetPrice(enc->high, i - kLenNumLowSymbols * 2, ProbPrices); + /* + unsigned sym = (i - kLenNumLowSymbols * 2) >> 1; + UInt32 price = a + RcTree_GetPrice(enc->high, kLenNumHighBits - 1, sym, ProbPrices); + UInt32 prob = enc->high[(1 << 7) + sym]; + prices[i ] = price + GET_PRICEa_0(prob); + prices[i + 1] = price + GET_PRICEa_1(prob); + */ + } + // } +} + +static void LenPriceEnc_UpdateTables(CLenPriceEnc *p, unsigned numPosStates, + const CLenEnc *enc, + const CProbPrice *ProbPrices) +{ + unsigned posState; for (posState = 0; posState < numPosStates; posState++) - LenPriceEnc_UpdateTable(p, posState, ProbPrices); -} - -static void LenEnc_Encode2(CLenPriceEnc *p, CRangeEnc *rc, UInt32 symbol, UInt32 posState, Bool updatePrice, const UInt32 *ProbPrices) -{ - LenEnc_Encode(&p->p, rc, symbol, posState); - if (updatePrice) - if (--p->counters[posState] == 0) - LenPriceEnc_UpdateTable(p, posState, ProbPrices); + LenPriceEnc_UpdateTable(p, posState, enc, ProbPrices); } - - -static void MovePos(CLzmaEnc *p, UInt32 num) -{ +/* #ifdef SHOW_STAT g_STAT_OFFSET += num; printf("\n MovePos %u", num); #endif +*/ - if (num != 0) - { - p->additionalOffset += num; - p->matchFinder.Skip(p->matchFinderObj, num); - } -} +#define MOVE_POS(p, num) { \ + p->additionalOffset += (num); \ + p->matchFinder.Skip(p->matchFinderObj, (num)); } -static UInt32 ReadMatchDistances(CLzmaEnc *p, UInt32 *numDistancePairsRes) + +static unsigned ReadMatchDistances(CLzmaEnc *p, unsigned *numPairsRes) { - UInt32 lenRes = 0, numPairs; + unsigned numPairs; + + p->additionalOffset++; p->numAvail = p->matchFinder.GetNumAvailableBytes(p->matchFinderObj); numPairs = p->matchFinder.GetMatches(p->matchFinderObj, p->matches); + *numPairsRes = numPairs; #ifdef SHOW_STAT printf("\n i = %u numPairs = %u ", g_STAT_OFFSET, numPairs / 2); g_STAT_OFFSET++; { - UInt32 i; + unsigned i; for (i = 0; i < numPairs; i += 2) printf("%2u %6u | ", p->matches[i], p->matches[i + 1]); } #endif - if (numPairs > 0) + if (numPairs == 0) + return 0; { - lenRes = p->matches[(size_t)numPairs - 2]; - if (lenRes == p->numFastBytes) + unsigned len = p->matches[(size_t)numPairs - 2]; + if (len != p->numFastBytes) + return len; { UInt32 numAvail = p->numAvail; if (numAvail > LZMA_MATCH_LEN_MAX) numAvail = LZMA_MATCH_LEN_MAX; { - const Byte *pbyCur = p->matchFinder.GetPointerToCurrentPos(p->matchFinderObj) - 1; - const Byte *pby = pbyCur + lenRes; + const Byte *p1 = p->matchFinder.GetPointerToCurrentPos(p->matchFinderObj) - 1; + const Byte *p2 = p1 + len; ptrdiff_t dif = (ptrdiff_t)-1 - p->matches[(size_t)numPairs - 1]; - const Byte *pbyLim = pbyCur + numAvail; - for (; pby != pbyLim && *pby == pby[dif]; pby++); - lenRes = (UInt32)(pby - pbyCur); + const Byte *lim = p1 + numAvail; + for (; p2 != lim && *p2 == p2[dif]; p2++); + return (unsigned)(p2 - p1); } } } - p->additionalOffset++; - *numDistancePairsRes = numPairs; - return lenRes; } +#define MARK_LIT ((UInt32)(Int32)-1) -#define MakeAsChar(p) (p)->backPrev = (UInt32)(-1); (p)->prev1IsChar = False; -#define MakeAsShortRep(p) (p)->backPrev = 0; (p)->prev1IsChar = False; -#define IsShortRep(p) ((p)->backPrev == 0) +#define MakeAs_Lit(p) { (p)->dist = MARK_LIT; (p)->extra = 0; } +#define MakeAs_ShortRep(p) { (p)->dist = 0; (p)->extra = 0; } +#define IsShortRep(p) ((p)->dist == 0) -static UInt32 GetRepLen1Price(CLzmaEnc *p, UInt32 state, UInt32 posState) -{ - return - GET_PRICE_0(p->isRepG0[state]) + - GET_PRICE_0(p->isRep0Long[state][posState]); -} -static UInt32 GetPureRepPrice(CLzmaEnc *p, UInt32 repIndex, UInt32 state, UInt32 posState) +#define GetPrice_ShortRep(p, state, posState) \ + ( GET_PRICE_0(p->isRepG0[state]) + GET_PRICE_0(p->isRep0Long[state][posState])) + +#define GetPrice_Rep_0(p, state, posState) ( \ + GET_PRICE_1(p->isMatch[state][posState]) \ + + GET_PRICE_1(p->isRep0Long[state][posState])) \ + + GET_PRICE_1(p->isRep[state]) \ + + GET_PRICE_0(p->isRepG0[state]) + + +static UInt32 GetPrice_PureRep(const CLzmaEnc *p, unsigned repIndex, size_t state, size_t posState) { UInt32 price; + UInt32 prob = p->isRepG0[state]; if (repIndex == 0) { - price = GET_PRICE_0(p->isRepG0[state]); + price = GET_PRICE_0(prob); price += GET_PRICE_1(p->isRep0Long[state][posState]); } else { - price = GET_PRICE_1(p->isRepG0[state]); + price = GET_PRICE_1(prob); + prob = p->isRepG1[state]; if (repIndex == 1) - price += GET_PRICE_0(p->isRepG1[state]); + price += GET_PRICE_0(prob); else { - price += GET_PRICE_1(p->isRepG1[state]); + price += GET_PRICE_1(prob); price += GET_PRICE(p->isRepG2[state], repIndex - 2); } } return price; } -static UInt32 GetRepPrice(CLzmaEnc *p, UInt32 repIndex, UInt32 len, UInt32 state, UInt32 posState) -{ - return p->repLenEnc.prices[posState][(size_t)len - LZMA_MATCH_LEN_MIN] + - GetPureRepPrice(p, repIndex, state, posState); -} -static UInt32 Backward(CLzmaEnc *p, UInt32 *backRes, UInt32 cur) +static unsigned Backward(CLzmaEnc *p, unsigned cur) { - UInt32 posMem = p->opt[cur].posPrev; - UInt32 backMem = p->opt[cur].backPrev; - p->optimumEndIndex = cur; - do + unsigned wr = cur + 1; + p->optEnd = wr; + + for (;;) { - if (p->opt[cur].prev1IsChar) + UInt32 dist = p->opt[cur].dist; + UInt32 len = p->opt[cur].len; + UInt32 extra = p->opt[cur].extra; + cur -= len; + + if (extra) { - MakeAsChar(&p->opt[posMem]) - p->opt[posMem].posPrev = posMem - 1; - if (p->opt[cur].prev2) + wr--; + p->opt[wr].len = len; + cur -= extra; + len = extra; + if (extra == 1) { - p->opt[(size_t)posMem - 1].prev1IsChar = False; - p->opt[(size_t)posMem - 1].posPrev = p->opt[cur].posPrev2; - p->opt[(size_t)posMem - 1].backPrev = p->opt[cur].backPrev2; + p->opt[wr].dist = dist; + dist = MARK_LIT; + } + else + { + p->opt[wr].dist = 0; + len--; + wr--; + p->opt[wr].dist = MARK_LIT; + p->opt[wr].len = 1; } } + + if (cur == 0) { - UInt32 posPrev = posMem; - UInt32 backCur = backMem; - - backMem = p->opt[posPrev].backPrev; - posMem = p->opt[posPrev].posPrev; - - p->opt[posPrev].backPrev = backCur; - p->opt[posPrev].posPrev = cur; - cur = posPrev; + p->backRes = dist; + p->optCur = wr; + return len; } + + wr--; + p->opt[wr].dist = dist; + p->opt[wr].len = len; } - while (cur != 0); - *backRes = p->opt[0].backPrev; - p->optimumCurrentIndex = p->opt[0].posPrev; - return p->optimumCurrentIndex; } -#define LIT_PROBS(pos, prevByte) (p->litProbs + ((((pos) & p->lpMask) << p->lc) + ((prevByte) >> (8 - p->lc))) * (UInt32)0x300) -static UInt32 GetOptimum(CLzmaEnc *p, UInt32 position, UInt32 *backRes) + +#define LIT_PROBS(pos, prevByte) \ + (p->litProbs + (UInt32)3 * (((((pos) << 8) + (prevByte)) & p->lpMask) << p->lc)) + + +static unsigned GetOptimum(CLzmaEnc *p, UInt32 position) { - UInt32 lenEnd, cur; - UInt32 reps[LZMA_NUM_REPS], repLens[LZMA_NUM_REPS]; + unsigned last, cur; + UInt32 reps[LZMA_NUM_REPS]; + unsigned repLens[LZMA_NUM_REPS]; UInt32 *matches; { - - UInt32 numAvail, mainLen, numPairs, repMaxIndex, i, posState, len; - UInt32 matchPrice, repMatchPrice, normalMatchPrice; - const Byte *data; - Byte curByte, matchByte; - - if (p->optimumEndIndex != p->optimumCurrentIndex) - { - const COptimal *opt = &p->opt[p->optimumCurrentIndex]; - UInt32 lenRes = opt->posPrev - p->optimumCurrentIndex; - *backRes = opt->backPrev; - p->optimumCurrentIndex = opt->posPrev; - return lenRes; - } - p->optimumCurrentIndex = p->optimumEndIndex = 0; - - if (p->additionalOffset == 0) - mainLen = ReadMatchDistances(p, &numPairs); - else - { - mainLen = p->longestMatchLength; - numPairs = p->numPairs; - } - - numAvail = p->numAvail; - if (numAvail < 2) - { - *backRes = (UInt32)(-1); - return 1; - } - if (numAvail > LZMA_MATCH_LEN_MAX) - numAvail = LZMA_MATCH_LEN_MAX; - - data = p->matchFinder.GetPointerToCurrentPos(p->matchFinderObj) - 1; - repMaxIndex = 0; - for (i = 0; i < LZMA_NUM_REPS; i++) - { - UInt32 lenTest; - const Byte *data2; - reps[i] = p->reps[i]; - data2 = data - reps[i] - 1; - if (data[0] != data2[0] || data[1] != data2[1]) + UInt32 numAvail; + unsigned numPairs, mainLen, repMaxIndex, i, posState; + UInt32 matchPrice, repMatchPrice; + const Byte *data; + Byte curByte, matchByte; + + p->optCur = p->optEnd = 0; + + if (p->additionalOffset == 0) + mainLen = ReadMatchDistances(p, &numPairs); + else { - repLens[i] = 0; - continue; + mainLen = p->longestMatchLen; + numPairs = p->numPairs; } - for (lenTest = 2; lenTest < numAvail && data[lenTest] == data2[lenTest]; lenTest++); - repLens[i] = lenTest; - if (lenTest > repLens[repMaxIndex]) - repMaxIndex = i; - } - if (repLens[repMaxIndex] >= p->numFastBytes) - { - UInt32 lenRes; - *backRes = repMaxIndex; - lenRes = repLens[repMaxIndex]; - MovePos(p, lenRes - 1); - return lenRes; - } - - matches = p->matches; - if (mainLen >= p->numFastBytes) - { - *backRes = matches[(size_t)numPairs - 1] + LZMA_NUM_REPS; - MovePos(p, mainLen - 1); - return mainLen; - } - curByte = *data; - matchByte = *(data - (reps[0] + 1)); - - if (mainLen < 2 && curByte != matchByte && repLens[repMaxIndex] < 2) - { - *backRes = (UInt32)-1; - return 1; - } - - p->opt[0].state = (CState)p->state; - - posState = (position & p->pbMask); - - { - const CLzmaProb *probs = LIT_PROBS(position, *(data - 1)); - p->opt[1].price = GET_PRICE_0(p->isMatch[p->state][posState]) + - (!IsCharState(p->state) ? - LitEnc_GetPriceMatched(probs, curByte, matchByte, p->ProbPrices) : + + numAvail = p->numAvail; + if (numAvail < 2) + { + p->backRes = MARK_LIT; + return 1; + } + if (numAvail > LZMA_MATCH_LEN_MAX) + numAvail = LZMA_MATCH_LEN_MAX; + + data = p->matchFinder.GetPointerToCurrentPos(p->matchFinderObj) - 1; + repMaxIndex = 0; + + for (i = 0; i < LZMA_NUM_REPS; i++) + { + unsigned len; + const Byte *data2; + reps[i] = p->reps[i]; + data2 = data - reps[i]; + if (data[0] != data2[0] || data[1] != data2[1]) + { + repLens[i] = 0; + continue; + } + for (len = 2; len < numAvail && data[len] == data2[len]; len++); + repLens[i] = len; + if (len > repLens[repMaxIndex]) + repMaxIndex = i; + } + + if (repLens[repMaxIndex] >= p->numFastBytes) + { + unsigned len; + p->backRes = repMaxIndex; + len = repLens[repMaxIndex]; + MOVE_POS(p, len - 1) + return len; + } + + matches = p->matches; + + if (mainLen >= p->numFastBytes) + { + p->backRes = matches[(size_t)numPairs - 1] + LZMA_NUM_REPS; + MOVE_POS(p, mainLen - 1) + return mainLen; + } + + curByte = *data; + matchByte = *(data - reps[0]); + + if (mainLen < 2 && curByte != matchByte && repLens[repMaxIndex] < 2) + { + p->backRes = MARK_LIT; + return 1; + } + + p->opt[0].state = (CState)p->state; + + posState = (position & p->pbMask); + + { + const CLzmaProb *probs = LIT_PROBS(position, *(data - 1)); + p->opt[1].price = GET_PRICE_0(p->isMatch[p->state][posState]) + + (!IsLitState(p->state) ? + LitEnc_Matched_GetPrice(probs, curByte, matchByte, p->ProbPrices) : LitEnc_GetPrice(probs, curByte, p->ProbPrices)); - } - - MakeAsChar(&p->opt[1]); - - matchPrice = GET_PRICE_1(p->isMatch[p->state][posState]); - repMatchPrice = matchPrice + GET_PRICE_1(p->isRep[p->state]); - - if (matchByte == curByte) - { - UInt32 shortRepPrice = repMatchPrice + GetRepLen1Price(p, p->state, posState); - if (shortRepPrice < p->opt[1].price) - { - p->opt[1].price = shortRepPrice; - MakeAsShortRep(&p->opt[1]); } - } - lenEnd = ((mainLen >= repLens[repMaxIndex]) ? mainLen : repLens[repMaxIndex]); - - if (lenEnd < 2) - { - *backRes = p->opt[1].backPrev; - return 1; - } - - p->opt[1].posPrev = 0; - for (i = 0; i < LZMA_NUM_REPS; i++) - p->opt[0].backs[i] = reps[i]; - - len = lenEnd; - do - p->opt[len--].price = kInfinityPrice; - while (len >= 2); - - for (i = 0; i < LZMA_NUM_REPS; i++) - { - UInt32 repLen = repLens[i]; - UInt32 price; - if (repLen < 2) - continue; - price = repMatchPrice + GetPureRepPrice(p, i, p->state, posState); - do + + MakeAs_Lit(&p->opt[1]); + + matchPrice = GET_PRICE_1(p->isMatch[p->state][posState]); + repMatchPrice = matchPrice + GET_PRICE_1(p->isRep[p->state]); + + if (matchByte == curByte) { - UInt32 curAndLenPrice = price + p->repLenEnc.prices[posState][(size_t)repLen - 2]; - COptimal *opt = &p->opt[repLen]; - if (curAndLenPrice < opt->price) + UInt32 shortRepPrice = repMatchPrice + GetPrice_ShortRep(p, p->state, posState); + if (shortRepPrice < p->opt[1].price) { - opt->price = curAndLenPrice; - opt->posPrev = 0; - opt->backPrev = i; - opt->prev1IsChar = False; + p->opt[1].price = shortRepPrice; + MakeAs_ShortRep(&p->opt[1]); } } - while (--repLen >= 2); - } - - normalMatchPrice = matchPrice + GET_PRICE_0(p->isRep[p->state]); - - len = ((repLens[0] >= 2) ? repLens[0] + 1 : 2); - if (len <= mainLen) - { - UInt32 offs = 0; - while (len > matches[offs]) - offs += 2; - for (; ; len++) + + last = (mainLen >= repLens[repMaxIndex] ? mainLen : repLens[repMaxIndex]); + + if (last < 2) { - COptimal *opt; - UInt32 distance = matches[(size_t)offs + 1]; + p->backRes = p->opt[1].dist; + return 1; + } + + p->opt[1].len = 1; + + p->opt[0].reps[0] = reps[0]; + p->opt[0].reps[1] = reps[1]; + p->opt[0].reps[2] = reps[2]; + p->opt[0].reps[3] = reps[3]; + + { + unsigned len = last; + do + p->opt[len--].price = kInfinityPrice; + while (len >= 2); + } - UInt32 curAndLenPrice = normalMatchPrice + p->lenEnc.prices[posState][(size_t)len - LZMA_MATCH_LEN_MIN]; - UInt32 lenToPosState = GetLenToPosState(len); - if (distance < kNumFullDistances) - curAndLenPrice += p->distancesPrices[lenToPosState][distance]; - else + // ---------- REP ---------- + + for (i = 0; i < LZMA_NUM_REPS; i++) + { + unsigned repLen = repLens[i]; + UInt32 price; + if (repLen < 2) + continue; + price = repMatchPrice + GetPrice_PureRep(p, i, p->state, posState); + do { - UInt32 slot; - GetPosSlot2(distance, slot); - curAndLenPrice += p->alignPrices[distance & kAlignMask] + p->posSlotPrices[lenToPosState][slot]; + UInt32 price2 = price + p->repLenEnc.prices[posState][(size_t)repLen - 2]; + COptimal *opt = &p->opt[repLen]; + if (price2 < opt->price) + { + opt->price = price2; + opt->len = repLen; + opt->dist = i; + opt->extra = 0; + } } - opt = &p->opt[len]; - if (curAndLenPrice < opt->price) + while (--repLen >= 2); + } + + + // ---------- MATCH ---------- + { + unsigned len = ((repLens[0] >= 2) ? repLens[0] + 1 : 2); + if (len <= mainLen) { - opt->price = curAndLenPrice; - opt->posPrev = 0; - opt->backPrev = distance + LZMA_NUM_REPS; - opt->prev1IsChar = False; - } - if (len == matches[offs]) - { - offs += 2; - if (offs == numPairs) - break; + unsigned offs = 0; + UInt32 normalMatchPrice = matchPrice + GET_PRICE_0(p->isRep[p->state]); + + while (len > matches[offs]) + offs += 2; + + for (; ; len++) + { + COptimal *opt; + UInt32 dist = matches[(size_t)offs + 1]; + UInt32 price2 = normalMatchPrice + p->lenEnc.prices[posState][(size_t)len - LZMA_MATCH_LEN_MIN]; + unsigned lenToPosState = GetLenToPosState(len); + + if (dist < kNumFullDistances) + price2 += p->distancesPrices[lenToPosState][dist & (kNumFullDistances - 1)]; + else + { + unsigned slot; + GetPosSlot2(dist, slot); + price2 += p->alignPrices[dist & kAlignMask]; + price2 += p->posSlotPrices[lenToPosState][slot]; + } + + opt = &p->opt[len]; + + if (price2 < opt->price) + { + opt->price = price2; + opt->len = len; + opt->dist = dist + LZMA_NUM_REPS; + opt->extra = 0; + } + + if (len == matches[offs]) + { + offs += 2; + if (offs == numPairs) + break; + } + } } } - } + - cur = 0; + cur = 0; #ifdef SHOW_STAT2 /* if (position >= 0) */ { unsigned i; printf("\n pos = %4X", position); - for (i = cur; i <= lenEnd; i++) + for (i = cur; i <= last; i++) printf("\nprice[%4X] = %u", position - cur + i, p->opt[i].price); } #endif - } + + + // ---------- Optimal Parsing ---------- + for (;;) { - UInt32 numAvail; - UInt32 numAvailFull, newLen, numPairs, posPrev, state, posState, startLen; - UInt32 curPrice, curAnd1Price, matchPrice, repMatchPrice; - Bool nextIsChar; + UInt32 numAvail, numAvailFull; + unsigned newLen, numPairs, prev, state, posState, startLen; + UInt32 curPrice, litPrice, matchPrice, repMatchPrice; + Bool nextIsLit; Byte curByte, matchByte; const Byte *data; - COptimal *curOpt; - COptimal *nextOpt; + COptimal *curOpt, *nextOpt; - cur++; - if (cur == lenEnd) - return Backward(p, backRes, cur); + if (++cur == last) + return Backward(p, cur); newLen = ReadMatchDistances(p, &numPairs); + if (newLen >= p->numFastBytes) { p->numPairs = numPairs; - p->longestMatchLength = newLen; - return Backward(p, backRes, cur); + p->longestMatchLen = newLen; + return Backward(p, cur); } - position++; + curOpt = &p->opt[cur]; - posPrev = curOpt->posPrev; - if (curOpt->prev1IsChar) - { - posPrev--; - if (curOpt->prev2) - { - state = p->opt[curOpt->posPrev2].state; - if (curOpt->backPrev2 < LZMA_NUM_REPS) - state = kRepNextStates[state]; - else - state = kMatchNextStates[state]; - } - else - state = p->opt[posPrev].state; - state = kLiteralNextStates[state]; - } - else - state = p->opt[posPrev].state; - if (posPrev == cur - 1) + prev = cur - curOpt->len; + + if (curOpt->len == 1) { + state = p->opt[prev].state; if (IsShortRep(curOpt)) state = kShortRepNextStates[state]; else @@ -1244,92 +1310,136 @@ static UInt32 GetOptimum(CLzmaEnc *p, UInt32 position, UInt32 *backRes) } else { - UInt32 pos; const COptimal *prevOpt; - if (curOpt->prev1IsChar && curOpt->prev2) + UInt32 b0; + UInt32 dist = curOpt->dist; + + if (curOpt->extra) { - posPrev = curOpt->posPrev2; - pos = curOpt->backPrev2; - state = kRepNextStates[state]; + prev -= curOpt->extra; + state = kState_RepAfterLit; + if (curOpt->extra == 1) + state = (dist < LZMA_NUM_REPS) ? kState_RepAfterLit : kState_MatchAfterLit; } else { - pos = curOpt->backPrev; - if (pos < LZMA_NUM_REPS) + state = p->opt[prev].state; + if (dist < LZMA_NUM_REPS) state = kRepNextStates[state]; else state = kMatchNextStates[state]; } - prevOpt = &p->opt[posPrev]; - if (pos < LZMA_NUM_REPS) + + prevOpt = &p->opt[prev]; + b0 = prevOpt->reps[0]; + + if (dist < LZMA_NUM_REPS) { - UInt32 i; - reps[0] = prevOpt->backs[pos]; - for (i = 1; i <= pos; i++) - reps[i] = prevOpt->backs[(size_t)i - 1]; - for (; i < LZMA_NUM_REPS; i++) - reps[i] = prevOpt->backs[i]; + if (dist == 0) + { + reps[0] = b0; + reps[1] = prevOpt->reps[1]; + reps[2] = prevOpt->reps[2]; + reps[3] = prevOpt->reps[3]; + } + else + { + reps[1] = b0; + b0 = prevOpt->reps[1]; + if (dist == 1) + { + reps[0] = b0; + reps[2] = prevOpt->reps[2]; + reps[3] = prevOpt->reps[3]; + } + else + { + reps[2] = b0; + reps[0] = prevOpt->reps[dist]; + reps[3] = prevOpt->reps[dist ^ 1]; + } + } } else { - UInt32 i; - reps[0] = (pos - LZMA_NUM_REPS); - for (i = 1; i < LZMA_NUM_REPS; i++) - reps[i] = prevOpt->backs[(size_t)i - 1]; + reps[0] = (dist - LZMA_NUM_REPS + 1); + reps[1] = b0; + reps[2] = prevOpt->reps[1]; + reps[3] = prevOpt->reps[2]; } } + curOpt->state = (CState)state; + curOpt->reps[0] = reps[0]; + curOpt->reps[1] = reps[1]; + curOpt->reps[2] = reps[2]; + curOpt->reps[3] = reps[3]; - curOpt->backs[0] = reps[0]; - curOpt->backs[1] = reps[1]; - curOpt->backs[2] = reps[2]; - curOpt->backs[3] = reps[3]; - - curPrice = curOpt->price; - nextIsChar = False; data = p->matchFinder.GetPointerToCurrentPos(p->matchFinderObj) - 1; curByte = *data; - matchByte = *(data - (reps[0] + 1)); + matchByte = *(data - reps[0]); + position++; posState = (position & p->pbMask); - curAnd1Price = curPrice + GET_PRICE_0(p->isMatch[state][posState]); - { - const CLzmaProb *probs = LIT_PROBS(position, *(data - 1)); - curAnd1Price += - (!IsCharState(state) ? - LitEnc_GetPriceMatched(probs, curByte, matchByte, p->ProbPrices) : - LitEnc_GetPrice(probs, curByte, p->ProbPrices)); - } + /* + The order of Price checks: + < LIT + <= SHORT_REP + < LIT : REP_0 + < REP [ : LIT : REP_0 ] + < MATCH [ : LIT : REP_0 ] + */ + + curPrice = curOpt->price; + litPrice = curPrice + GET_PRICE_0(p->isMatch[state][posState]); nextOpt = &p->opt[(size_t)cur + 1]; + nextIsLit = False; - if (curAnd1Price < nextOpt->price) + // if (litPrice >= nextOpt->price) litPrice = 0; else // 18.new { - nextOpt->price = curAnd1Price; - nextOpt->posPrev = cur; - MakeAsChar(nextOpt); - nextIsChar = True; + const CLzmaProb *probs = LIT_PROBS(position, *(data - 1)); + litPrice += (!IsLitState(state) ? + LitEnc_Matched_GetPrice(probs, curByte, matchByte, p->ProbPrices) : + LitEnc_GetPrice(probs, curByte, p->ProbPrices)); + + if (litPrice < nextOpt->price) + { + nextOpt->price = litPrice; + nextOpt->len = 1; + MakeAs_Lit(nextOpt); + nextIsLit = True; + } } matchPrice = curPrice + GET_PRICE_1(p->isMatch[state][posState]); repMatchPrice = matchPrice + GET_PRICE_1(p->isRep[state]); - if (matchByte == curByte && !(nextOpt->posPrev < cur && nextOpt->backPrev == 0)) + // ---------- SHORT_REP ---------- + // if (IsLitState(state)) // 18.new + if (matchByte == curByte) + // if (repMatchPrice < nextOpt->price) // 18.new + if (nextOpt->len < 2 + || (nextOpt->dist != 0 + && nextOpt->extra <= 1 // 17.old + )) { - UInt32 shortRepPrice = repMatchPrice + GetRepLen1Price(p, state, posState); - if (shortRepPrice <= nextOpt->price) + UInt32 shortRepPrice = repMatchPrice + GetPrice_ShortRep(p, state, posState); + if (shortRepPrice <= nextOpt->price) // 17.old + // if (shortRepPrice < nextOpt->price) // 18.new { nextOpt->price = shortRepPrice; - nextOpt->posPrev = cur; - MakeAsShortRep(nextOpt); - nextIsChar = True; + nextOpt->len = 1; + MakeAs_ShortRep(nextOpt); + nextIsLit = False; } } + numAvailFull = p->numAvail; { UInt32 temp = kNumOpts - 1 - cur; - if (temp < numAvailFull) + if (numAvailFull > temp) numAvailFull = temp; } @@ -1337,41 +1447,53 @@ static UInt32 GetOptimum(CLzmaEnc *p, UInt32 position, UInt32 *backRes) continue; numAvail = (numAvailFull <= p->numFastBytes ? numAvailFull : p->numFastBytes); - if (!nextIsChar && matchByte != curByte) /* speed optimization */ - { - /* try Literal + rep0 */ - UInt32 temp; - UInt32 lenTest2; - const Byte *data2 = data - reps[0] - 1; - UInt32 limit = p->numFastBytes + 1; - if (limit > numAvailFull) - limit = numAvailFull; + // numAvail <= p->numFastBytes - for (temp = 1; temp < limit && data[temp] == data2[temp]; temp++); - lenTest2 = temp - 1; - if (lenTest2 >= 2) + // ---------- LIT : REP_0 ---------- + + if ( + // litPrice != 0 && // 18.new + !nextIsLit + && matchByte != curByte + && numAvailFull > 2) + { + const Byte *data2 = data - reps[0]; + if (data[1] == data2[1] && data[2] == data2[2]) { - UInt32 state2 = kLiteralNextStates[state]; - UInt32 posStateNext = (position + 1) & p->pbMask; - UInt32 nextRepMatchPrice = curAnd1Price + - GET_PRICE_1(p->isMatch[state2][posStateNext]) + - GET_PRICE_1(p->isRep[state2]); - /* for (; lenTest2 >= 2; lenTest2--) */ + unsigned len; + unsigned limit = p->numFastBytes + 1; + if (limit > numAvailFull) + limit = numAvailFull; + for (len = 3; len < limit && data[len] == data2[len]; len++); + { - UInt32 curAndLenPrice; - COptimal *opt; - UInt32 offset = cur + 1 + lenTest2; - while (lenEnd < offset) - p->opt[++lenEnd].price = kInfinityPrice; - curAndLenPrice = nextRepMatchPrice + GetRepPrice(p, 0, lenTest2, state2, posStateNext); - opt = &p->opt[offset]; - if (curAndLenPrice < opt->price) + unsigned state2 = kLiteralNextStates[state]; + unsigned posState2 = (position + 1) & p->pbMask; + UInt32 price = litPrice + GetPrice_Rep_0(p, state2, posState2); { - opt->price = curAndLenPrice; - opt->posPrev = cur + 1; - opt->backPrev = 0; - opt->prev1IsChar = True; - opt->prev2 = False; + unsigned offset = cur + len; + while (last < offset) + p->opt[++last].price = kInfinityPrice; + + // do + { + UInt32 price2; + COptimal *opt; + len--; + // price2 = price + GetPrice_Len_Rep_0(p, len, state2, posState2); + price2 = price + p->repLenEnc.prices[posState2][len - LZMA_MATCH_LEN_MIN]; + + opt = &p->opt[offset]; + // offset--; + if (price2 < opt->price) + { + opt->price = price2; + opt->len = len; + opt->dist = 0; + opt->extra = 1; + } + } + // while (len >= 3); } } } @@ -1379,87 +1501,105 @@ static UInt32 GetOptimum(CLzmaEnc *p, UInt32 position, UInt32 *backRes) startLen = 2; /* speed optimization */ { - UInt32 repIndex; - for (repIndex = 0; repIndex < LZMA_NUM_REPS; repIndex++) - { - UInt32 lenTest; - UInt32 lenTestTemp; - UInt32 price; - const Byte *data2 = data - reps[repIndex] - 1; - if (data[0] != data2[0] || data[1] != data2[1]) - continue; - for (lenTest = 2; lenTest < numAvail && data[lenTest] == data2[lenTest]; lenTest++); - while (lenEnd < cur + lenTest) - p->opt[++lenEnd].price = kInfinityPrice; - lenTestTemp = lenTest; - price = repMatchPrice + GetPureRepPrice(p, repIndex, state, posState); - do + // ---------- REP ---------- + unsigned repIndex = 0; // 17.old + // unsigned repIndex = IsLitState(state) ? 0 : 1; // 18.notused + for (; repIndex < LZMA_NUM_REPS; repIndex++) { - UInt32 curAndLenPrice = price + p->repLenEnc.prices[posState][(size_t)lenTest - 2]; - COptimal *opt = &p->opt[cur + lenTest]; - if (curAndLenPrice < opt->price) - { - opt->price = curAndLenPrice; - opt->posPrev = cur; - opt->backPrev = repIndex; - opt->prev1IsChar = False; - } - } - while (--lenTest >= 2); - lenTest = lenTestTemp; - - if (repIndex == 0) - startLen = lenTest + 1; + unsigned len; + UInt32 price; + const Byte *data2 = data - reps[repIndex]; + if (data[0] != data2[0] || data[1] != data2[1]) + continue; - /* if (_maxMode) */ + for (len = 2; len < numAvail && data[len] == data2[len]; len++); + + // if (len < startLen) continue; // 18.new: speed optimization + + while (last < cur + len) + p->opt[++last].price = kInfinityPrice; { - UInt32 lenTest2 = lenTest + 1; - UInt32 limit = lenTest2 + p->numFastBytes; + unsigned len2 = len; + price = repMatchPrice + GetPrice_PureRep(p, repIndex, state, posState); + do + { + UInt32 price2 = price + p->repLenEnc.prices[posState][(size_t)len2 - 2]; + COptimal *opt = &p->opt[cur + len2]; + if (price2 < opt->price) + { + opt->price = price2; + opt->len = len2; + opt->dist = repIndex; + opt->extra = 0; + } + } + while (--len2 >= 2); + } + + if (repIndex == 0) startLen = len + 1; // 17.old + // startLen = len + 1; // 18.new + + /* if (_maxMode) */ + { + // ---------- REP : LIT : REP_0 ---------- + // numFastBytes + 1 + numFastBytes + + unsigned len2 = len + 1; + unsigned limit = len2 + p->numFastBytes; if (limit > numAvailFull) limit = numAvailFull; - for (; lenTest2 < limit && data[lenTest2] == data2[lenTest2]; lenTest2++); - lenTest2 -= lenTest + 1; - if (lenTest2 >= 2) + + for (; len2 < limit && data[len2] == data2[len2]; len2++); + + len2 -= len; + if (len2 >= 3) { - UInt32 nextRepMatchPrice; - UInt32 state2 = kRepNextStates[state]; - UInt32 posStateNext = (position + lenTest) & p->pbMask; - UInt32 curAndLenCharPrice = - price + p->repLenEnc.prices[posState][(size_t)lenTest - 2] + - GET_PRICE_0(p->isMatch[state2][posStateNext]) + - LitEnc_GetPriceMatched(LIT_PROBS(position + lenTest, data[(size_t)lenTest - 1]), - data[lenTest], data2[lenTest], p->ProbPrices); - state2 = kLiteralNextStates[state2]; - posStateNext = (position + lenTest + 1) & p->pbMask; - nextRepMatchPrice = curAndLenCharPrice + - GET_PRICE_1(p->isMatch[state2][posStateNext]) + - GET_PRICE_1(p->isRep[state2]); + unsigned state2 = kRepNextStates[state]; + unsigned posState2 = (position + len) & p->pbMask; + price += + p->repLenEnc.prices[posState][(size_t)len - 2] + + GET_PRICE_0(p->isMatch[state2][posState2]) + + LitEnc_Matched_GetPrice(LIT_PROBS(position + len, data[(size_t)len - 1]), + data[len], data2[len], p->ProbPrices); - /* for (; lenTest2 >= 2; lenTest2--) */ + // state2 = kLiteralNextStates[state2]; + state2 = kState_LitAfterRep; + posState2 = (posState2 + 1) & p->pbMask; + + + price += GetPrice_Rep_0(p, state2, posState2); { - UInt32 curAndLenPrice; - COptimal *opt; - UInt32 offset = cur + lenTest + 1 + lenTest2; - while (lenEnd < offset) - p->opt[++lenEnd].price = kInfinityPrice; - curAndLenPrice = nextRepMatchPrice + GetRepPrice(p, 0, lenTest2, state2, posStateNext); - opt = &p->opt[offset]; - if (curAndLenPrice < opt->price) + unsigned offset = cur + len + len2; + while (last < offset) + p->opt[++last].price = kInfinityPrice; + // do { - opt->price = curAndLenPrice; - opt->posPrev = cur + lenTest + 1; - opt->backPrev = 0; - opt->prev1IsChar = True; - opt->prev2 = True; - opt->posPrev2 = cur; - opt->backPrev2 = repIndex; + unsigned price2; + COptimal *opt; + len2--; + // price2 = price + GetPrice_Len_Rep_0(p, len2, state2, posState2); + price2 = price + p->repLenEnc.prices[posState2][len2 - LZMA_MATCH_LEN_MIN]; + + opt = &p->opt[offset]; + // offset--; + if (price2 < opt->price) + { + opt->price = price2; + opt->len = len2; + opt->extra = (CExtra)(len + 1); + opt->dist = repIndex; + } } + // while (len2 >= 3); } } } + } } - } - /* for (UInt32 lenTest = 2; lenTest <= newLen; lenTest++) */ + + + // ---------- MATCH ---------- + /* for (unsigned len = 2; len <= newLen; len++) */ if (newLen > numAvail) { newLen = numAvail; @@ -1467,134 +1607,148 @@ static UInt32 GetOptimum(CLzmaEnc *p, UInt32 position, UInt32 *backRes) matches[numPairs] = newLen; numPairs += 2; } + if (newLen >= startLen) { UInt32 normalMatchPrice = matchPrice + GET_PRICE_0(p->isRep[state]); - UInt32 offs, curBack, posSlot; - UInt32 lenTest; - while (lenEnd < cur + newLen) - p->opt[++lenEnd].price = kInfinityPrice; + UInt32 dist; + unsigned offs, posSlot, len; + while (last < cur + newLen) + p->opt[++last].price = kInfinityPrice; offs = 0; while (startLen > matches[offs]) offs += 2; - curBack = matches[(size_t)offs + 1]; - GetPosSlot2(curBack, posSlot); - for (lenTest = /*2*/ startLen; ; lenTest++) + dist = matches[(size_t)offs + 1]; + + // if (dist >= kNumFullDistances) + GetPosSlot2(dist, posSlot); + + for (len = /*2*/ startLen; ; len++) { - UInt32 curAndLenPrice = normalMatchPrice + p->lenEnc.prices[posState][(size_t)lenTest - LZMA_MATCH_LEN_MIN]; + UInt32 price = normalMatchPrice + p->lenEnc.prices[posState][(size_t)len - LZMA_MATCH_LEN_MIN]; { - UInt32 lenToPosState = GetLenToPosState(lenTest); - COptimal *opt; - if (curBack < kNumFullDistances) - curAndLenPrice += p->distancesPrices[lenToPosState][curBack]; - else - curAndLenPrice += p->posSlotPrices[lenToPosState][posSlot] + p->alignPrices[curBack & kAlignMask]; - - opt = &p->opt[cur + lenTest]; - if (curAndLenPrice < opt->price) - { - opt->price = curAndLenPrice; - opt->posPrev = cur; - opt->backPrev = curBack + LZMA_NUM_REPS; - opt->prev1IsChar = False; - } + COptimal *opt; + unsigned lenToPosState = len - 2; lenToPosState = GetLenToPosState2(lenToPosState); + if (dist < kNumFullDistances) + price += p->distancesPrices[lenToPosState][dist & (kNumFullDistances - 1)]; + else + price += p->posSlotPrices[lenToPosState][posSlot] + p->alignPrices[dist & kAlignMask]; + + opt = &p->opt[cur + len]; + if (price < opt->price) + { + opt->price = price; + opt->len = len; + opt->dist = dist + LZMA_NUM_REPS; + opt->extra = 0; + } } - if (/*_maxMode && */lenTest == matches[offs]) + if (/*_maxMode && */ len == matches[offs]) { - /* Try Match + Literal + Rep0 */ - const Byte *data2 = data - curBack - 1; - UInt32 lenTest2 = lenTest + 1; - UInt32 limit = lenTest2 + p->numFastBytes; + // MATCH : LIT : REP_0 + + const Byte *data2 = data - dist - 1; + unsigned len2 = len + 1; + unsigned limit = len2 + p->numFastBytes; if (limit > numAvailFull) limit = numAvailFull; - for (; lenTest2 < limit && data[lenTest2] == data2[lenTest2]; lenTest2++); - lenTest2 -= lenTest + 1; - if (lenTest2 >= 2) + + for (; len2 < limit && data[len2] == data2[len2]; len2++); + + len2 -= len; + + if (len2 >= 3) { - UInt32 nextRepMatchPrice; - UInt32 state2 = kMatchNextStates[state]; - UInt32 posStateNext = (position + lenTest) & p->pbMask; - UInt32 curAndLenCharPrice = curAndLenPrice + - GET_PRICE_0(p->isMatch[state2][posStateNext]) + - LitEnc_GetPriceMatched(LIT_PROBS(position + lenTest, data[(size_t)lenTest - 1]), - data[lenTest], data2[lenTest], p->ProbPrices); - state2 = kLiteralNextStates[state2]; - posStateNext = (posStateNext + 1) & p->pbMask; - nextRepMatchPrice = curAndLenCharPrice + - GET_PRICE_1(p->isMatch[state2][posStateNext]) + - GET_PRICE_1(p->isRep[state2]); - - /* for (; lenTest2 >= 2; lenTest2--) */ + unsigned state2 = kMatchNextStates[state]; + unsigned posState2 = (position + len) & p->pbMask; + unsigned offset; + price += GET_PRICE_0(p->isMatch[state2][posState2]); + price += LitEnc_Matched_GetPrice(LIT_PROBS(position + len, data[(size_t)len - 1]), + data[len], data2[len], p->ProbPrices); + + // state2 = kLiteralNextStates[state2]; + state2 = kState_LitAfterMatch; + + posState2 = (posState2 + 1) & p->pbMask; + price += GetPrice_Rep_0(p, state2, posState2); + + offset = cur + len + len2; + while (last < offset) + p->opt[++last].price = kInfinityPrice; + // do { - UInt32 offset = cur + lenTest + 1 + lenTest2; - UInt32 curAndLenPrice2; + UInt32 price2; COptimal *opt; - while (lenEnd < offset) - p->opt[++lenEnd].price = kInfinityPrice; - curAndLenPrice2 = nextRepMatchPrice + GetRepPrice(p, 0, lenTest2, state2, posStateNext); + len2--; + // price2 = price + GetPrice_Len_Rep_0(p, len2, state2, posState2); + price2 = price + p->repLenEnc.prices[posState2][len2 - LZMA_MATCH_LEN_MIN]; opt = &p->opt[offset]; - if (curAndLenPrice2 < opt->price) + // offset--; + if (price2 < opt->price) { - opt->price = curAndLenPrice2; - opt->posPrev = cur + lenTest + 1; - opt->backPrev = 0; - opt->prev1IsChar = True; - opt->prev2 = True; - opt->posPrev2 = cur; - opt->backPrev2 = curBack + LZMA_NUM_REPS; + opt->price = price2; + opt->len = len2; + opt->extra = (CExtra)(len + 1); + opt->dist = dist + LZMA_NUM_REPS; } } + // while (len2 >= 3); } + offs += 2; if (offs == numPairs) break; - curBack = matches[(size_t)offs + 1]; - if (curBack >= kNumFullDistances) - GetPosSlot2(curBack, posSlot); + dist = matches[(size_t)offs + 1]; + // if (dist >= kNumFullDistances) + GetPosSlot2(dist, posSlot); } } } } } + + #define ChangePair(smallDist, bigDist) (((bigDist) >> 7) > (smallDist)) -static UInt32 GetOptimumFast(CLzmaEnc *p, UInt32 *backRes) + + +static unsigned GetOptimumFast(CLzmaEnc *p) { - UInt32 numAvail, mainLen, mainDist, numPairs, repIndex, repLen, i; + UInt32 numAvail, mainDist; + unsigned mainLen, numPairs, repIndex, repLen, i; const Byte *data; - const UInt32 *matches; if (p->additionalOffset == 0) mainLen = ReadMatchDistances(p, &numPairs); else { - mainLen = p->longestMatchLength; + mainLen = p->longestMatchLen; numPairs = p->numPairs; } numAvail = p->numAvail; - *backRes = (UInt32)-1; + p->backRes = MARK_LIT; if (numAvail < 2) return 1; if (numAvail > LZMA_MATCH_LEN_MAX) numAvail = LZMA_MATCH_LEN_MAX; data = p->matchFinder.GetPointerToCurrentPos(p->matchFinderObj) - 1; - repLen = repIndex = 0; + for (i = 0; i < LZMA_NUM_REPS; i++) { - UInt32 len; - const Byte *data2 = data - p->reps[i] - 1; + unsigned len; + const Byte *data2 = data - p->reps[i]; if (data[0] != data2[0] || data[1] != data2[1]) continue; for (len = 2; len < numAvail && data[len] == data2[len]; len++); if (len >= p->numFastBytes) { - *backRes = i; - MovePos(p, len - 1); + p->backRes = i; + MOVE_POS(p, len - 1) return len; } if (len > repLen) @@ -1604,84 +1758,152 @@ static UInt32 GetOptimumFast(CLzmaEnc *p, UInt32 *backRes) } } - matches = p->matches; if (mainLen >= p->numFastBytes) { - *backRes = matches[(size_t)numPairs - 1] + LZMA_NUM_REPS; - MovePos(p, mainLen - 1); + p->backRes = p->matches[(size_t)numPairs - 1] + LZMA_NUM_REPS; + MOVE_POS(p, mainLen - 1) return mainLen; } mainDist = 0; /* for GCC */ + if (mainLen >= 2) { - mainDist = matches[(size_t)numPairs - 1]; - while (numPairs > 2 && mainLen == matches[(size_t)numPairs - 4] + 1) + mainDist = p->matches[(size_t)numPairs - 1]; + while (numPairs > 2) { - if (!ChangePair(matches[(size_t)numPairs - 3], mainDist)) + UInt32 dist2; + if (mainLen != p->matches[(size_t)numPairs - 4] + 1) + break; + dist2 = p->matches[(size_t)numPairs - 3]; + if (!ChangePair(dist2, mainDist)) break; numPairs -= 2; - mainLen = matches[(size_t)numPairs - 2]; - mainDist = matches[(size_t)numPairs - 1]; + mainLen--; + mainDist = dist2; } if (mainLen == 2 && mainDist >= 0x80) mainLen = 1; } - if (repLen >= 2 && ( - (repLen + 1 >= mainLen) || - (repLen + 2 >= mainLen && mainDist >= (1 << 9)) || - (repLen + 3 >= mainLen && mainDist >= (1 << 15)))) + if (repLen >= 2) + if ( repLen + 1 >= mainLen + || (repLen + 2 >= mainLen && mainDist >= (1 << 9)) + || (repLen + 3 >= mainLen && mainDist >= (1 << 15))) { - *backRes = repIndex; - MovePos(p, repLen - 1); + p->backRes = repIndex; + MOVE_POS(p, repLen - 1) return repLen; } if (mainLen < 2 || numAvail <= 2) return 1; - p->longestMatchLength = ReadMatchDistances(p, &p->numPairs); - if (p->longestMatchLength >= 2) { - UInt32 newDistance = matches[(size_t)p->numPairs - 1]; - if ((p->longestMatchLength >= mainLen && newDistance < mainDist) || - (p->longestMatchLength == mainLen + 1 && !ChangePair(mainDist, newDistance)) || - (p->longestMatchLength > mainLen + 1) || - (p->longestMatchLength + 1 >= mainLen && mainLen >= 3 && ChangePair(newDistance, mainDist))) - return 1; + unsigned len1 = ReadMatchDistances(p, &p->numPairs); + p->longestMatchLen = len1; + + if (len1 >= 2) + { + UInt32 newDist = p->matches[(size_t)p->numPairs - 1]; + if ( (len1 >= mainLen && newDist < mainDist) + || (len1 == mainLen + 1 && !ChangePair(mainDist, newDist)) + || (len1 > mainLen + 1) + || (len1 + 1 >= mainLen && mainLen >= 3 && ChangePair(newDist, mainDist))) + return 1; + } } data = p->matchFinder.GetPointerToCurrentPos(p->matchFinderObj) - 1; + for (i = 0; i < LZMA_NUM_REPS; i++) { - UInt32 len, limit; - const Byte *data2 = data - p->reps[i] - 1; + unsigned len, limit; + const Byte *data2 = data - p->reps[i]; if (data[0] != data2[0] || data[1] != data2[1]) continue; limit = mainLen - 1; - for (len = 2; len < limit && data[len] == data2[len]; len++); - if (len >= limit) - return 1; + for (len = 2;; len++) + { + if (len >= limit) + return 1; + if (data[len] != data2[len]) + break; + } + } + + p->backRes = mainDist + LZMA_NUM_REPS; + if (mainLen != 2) + { + MOVE_POS(p, mainLen - 2) } - *backRes = mainDist + LZMA_NUM_REPS; - MovePos(p, mainLen - 2); return mainLen; } -static void WriteEndMarker(CLzmaEnc *p, UInt32 posState) + + + +static void WriteEndMarker(CLzmaEnc *p, unsigned posState) { - UInt32 len; - RangeEnc_EncodeBit(&p->rc, &p->isMatch[p->state][posState], 1); - RangeEnc_EncodeBit(&p->rc, &p->isRep[p->state], 0); + UInt32 range; + range = p->rc.range; + { + UInt32 ttt, newBound; + CLzmaProb *prob = &p->isMatch[p->state][posState]; + RC_BIT_PRE(&p->rc, prob) + RC_BIT_1(&p->rc, prob) + prob = &p->isRep[p->state]; + RC_BIT_PRE(&p->rc, prob) + RC_BIT_0(&p->rc, prob) + } p->state = kMatchNextStates[p->state]; - len = LZMA_MATCH_LEN_MIN; - LenEnc_Encode2(&p->lenEnc, &p->rc, len - LZMA_MATCH_LEN_MIN, posState, !p->fastMode, p->ProbPrices); - RcTree_Encode(&p->rc, p->posSlotEncoder[GetLenToPosState(len)], kNumPosSlotBits, (1 << kNumPosSlotBits) - 1); - RangeEnc_EncodeDirectBits(&p->rc, (((UInt32)1 << 30) - 1) >> kNumAlignBits, 30 - kNumAlignBits); - RcTree_ReverseEncode(&p->rc, p->posAlignEncoder, kNumAlignBits, kAlignMask); + + p->rc.range = range; + LenEnc_Encode(&p->lenProbs, &p->rc, 0, posState); + range = p->rc.range; + + { + // RcTree_Encode_PosSlot(&p->rc, p->posSlotEncoder[0], (1 << kNumPosSlotBits) - 1); + CLzmaProb *probs = p->posSlotEncoder[0]; + unsigned m = 1; + do + { + UInt32 ttt, newBound; + RC_BIT_PRE(p, probs + m) + RC_BIT_1(&p->rc, probs + m); + m = (m << 1) + 1; + } + while (m < (1 << kNumPosSlotBits)); + } + { + // RangeEnc_EncodeDirectBits(&p->rc, ((UInt32)1 << (30 - kNumAlignBits)) - 1, 30 - kNumAlignBits); UInt32 range = p->range; + unsigned numBits = 30 - kNumAlignBits; + do + { + range >>= 1; + p->rc.low += range; + RC_NORM(&p->rc) + } + while (--numBits); + } + + { + // RcTree_ReverseEncode(&p->rc, p->posAlignEncoder, kNumAlignBits, kAlignMask); + CLzmaProb *probs = p->posAlignEncoder; + unsigned m = 1; + do + { + UInt32 ttt, newBound; + RC_BIT_PRE(p, probs + m) + RC_BIT_1(&p->rc, probs + m); + m = (m << 1) + 1; + } + while (m < kAlignTableSize); + } + p->rc.range = range; } + static SRes CheckErrors(CLzmaEnc *p) { if (p->result != SZ_OK) @@ -1695,7 +1917,8 @@ static SRes CheckErrors(CLzmaEnc *p) return p->result; } -static SRes Flush(CLzmaEnc *p, UInt32 nowPos) + +MY_NO_INLINE static SRes Flush(CLzmaEnc *p, UInt32 nowPos) { /* ReleaseMFStream(); */ p->finished = True; @@ -1706,47 +1929,108 @@ static SRes Flush(CLzmaEnc *p, UInt32 nowPos) return CheckErrors(p); } + + static void FillAlignPrices(CLzmaEnc *p) { - UInt32 i; - for (i = 0; i < kAlignTableSize; i++) - p->alignPrices[i] = RcTree_ReverseGetPrice(p->posAlignEncoder, kNumAlignBits, i, p->ProbPrices); + unsigned i; + const CProbPrice *ProbPrices = p->ProbPrices; + const CLzmaProb *probs = p->posAlignEncoder; p->alignPriceCount = 0; + for (i = 0; i < kAlignTableSize / 2; i++) + { + UInt32 price = 0; + unsigned symbol = i; + unsigned m = 1; + unsigned bit; + UInt32 prob; + bit = symbol & 1; symbol >>= 1; price += GET_PRICEa(probs[m], bit); m = (m << 1) + bit; + bit = symbol & 1; symbol >>= 1; price += GET_PRICEa(probs[m], bit); m = (m << 1) + bit; + bit = symbol & 1; symbol >>= 1; price += GET_PRICEa(probs[m], bit); m = (m << 1) + bit; + prob = probs[m]; + p->alignPrices[i ] = price + GET_PRICEa_0(prob); + p->alignPrices[i + 8] = price + GET_PRICEa_1(prob); + // p->alignPrices[i] = RcTree_ReverseGetPrice(p->posAlignEncoder, kNumAlignBits, i, p->ProbPrices); + } } + static void FillDistancesPrices(CLzmaEnc *p) { UInt32 tempPrices[kNumFullDistances]; - UInt32 i, lenToPosState; + unsigned i, lenToPosState; + + const CProbPrice *ProbPrices = p->ProbPrices; + p->matchPriceCount = 0; + for (i = kStartPosModelIndex; i < kNumFullDistances; i++) { - UInt32 posSlot = GetPosSlot1(i); - UInt32 footerBits = ((posSlot >> 1) - 1); - UInt32 base = ((2 | (posSlot & 1)) << footerBits); - tempPrices[i] = RcTree_ReverseGetPrice(p->posEncoders + base - posSlot - 1, footerBits, i - base, p->ProbPrices); + unsigned posSlot = GetPosSlot1(i); + unsigned footerBits = ((posSlot >> 1) - 1); + unsigned base = ((2 | (posSlot & 1)) << footerBits); + // tempPrices[i] = RcTree_ReverseGetPrice(p->posEncoders + base, footerBits, i - base, p->ProbPrices); + + const CLzmaProb *probs = p->posEncoders + base; + UInt32 price = 0; + unsigned m = 1; + unsigned symbol = i - base; + do + { + unsigned bit = symbol & 1; + symbol >>= 1; + price += GET_PRICEa(probs[m], bit); + m = (m << 1) + bit; + } + while (--footerBits); + tempPrices[i] = price; } for (lenToPosState = 0; lenToPosState < kNumLenToPosStates; lenToPosState++) { - UInt32 posSlot; + unsigned posSlot; const CLzmaProb *encoder = p->posSlotEncoder[lenToPosState]; UInt32 *posSlotPrices = p->posSlotPrices[lenToPosState]; - for (posSlot = 0; posSlot < p->distTableSize; posSlot++) - posSlotPrices[posSlot] = RcTree_GetPrice(encoder, kNumPosSlotBits, posSlot, p->ProbPrices); - for (posSlot = kEndPosModelIndex; posSlot < p->distTableSize; posSlot++) - posSlotPrices[posSlot] += ((((posSlot >> 1) - 1) - kNumAlignBits) << kNumBitPriceShiftBits); + unsigned distTableSize = p->distTableSize; + const CLzmaProb *probs = encoder; + for (posSlot = 0; posSlot < distTableSize; posSlot += 2) + { + // posSlotPrices[posSlot] = RcTree_GetPrice(encoder, kNumPosSlotBits, posSlot, p->ProbPrices); + UInt32 price = 0; + unsigned bit; + unsigned symbol = (posSlot >> 1) + (1 << (kNumPosSlotBits - 1)); + UInt32 prob; + bit = symbol & 1; symbol >>= 1; price += GET_PRICEa(probs[symbol], bit); + bit = symbol & 1; symbol >>= 1; price += GET_PRICEa(probs[symbol], bit); + bit = symbol & 1; symbol >>= 1; price += GET_PRICEa(probs[symbol], bit); + bit = symbol & 1; symbol >>= 1; price += GET_PRICEa(probs[symbol], bit); + bit = symbol & 1; symbol >>= 1; price += GET_PRICEa(probs[symbol], bit); + prob = probs[(posSlot >> 1) + (1 << (kNumPosSlotBits - 1))]; + posSlotPrices[posSlot ] = price + GET_PRICEa_0(prob); + posSlotPrices[posSlot + 1] = price + GET_PRICEa_1(prob); + } + for (posSlot = kEndPosModelIndex; posSlot < distTableSize; posSlot++) + posSlotPrices[posSlot] += ((UInt32)(((posSlot >> 1) - 1) - kNumAlignBits) << kNumBitPriceShiftBits); { UInt32 *distancesPrices = p->distancesPrices[lenToPosState]; - for (i = 0; i < kStartPosModelIndex; i++) - distancesPrices[i] = posSlotPrices[i]; - for (; i < kNumFullDistances; i++) - distancesPrices[i] = posSlotPrices[GetPosSlot1(i)] + tempPrices[i]; + { + distancesPrices[0] = posSlotPrices[0]; + distancesPrices[1] = posSlotPrices[1]; + distancesPrices[2] = posSlotPrices[2]; + distancesPrices[3] = posSlotPrices[3]; + } + for (i = 4; i < kNumFullDistances; i += 2) + { + UInt32 slotPrice = posSlotPrices[GetPosSlot1(i)]; + distancesPrices[i ] = slotPrice + tempPrices[i]; + distancesPrices[i + 1] = slotPrice + tempPrices[i + 1]; + } } } - p->matchPriceCount = 0; } + + void LzmaEnc_Construct(CLzmaEnc *p) { RangeEnc_Construct(&p->rc); @@ -1770,6 +2054,7 @@ void LzmaEnc_Construct(CLzmaEnc *p) LzmaEnc_InitPriceTables(p->ProbPrices); p->litProbs = NULL; p->saveState.litProbs = NULL; + } CLzmaEncHandle LzmaEnc_Create(ISzAllocPtr alloc) @@ -1806,7 +2091,8 @@ void LzmaEnc_Destroy(CLzmaEncHandle p, ISzAllocPtr alloc, ISzAllocPtr allocBig) ISzAlloc_Free(alloc, p); } -static SRes LzmaEnc_CodeOneBlock(CLzmaEnc *p, Bool useLimits, UInt32 maxPackSize, UInt32 maxUnpackSize) + +static SRes LzmaEnc_CodeOneBlock(CLzmaEnc *p, UInt32 maxPackSize, UInt32 maxUnpackSize) { UInt32 nowPos32, startPos32; if (p->needInit) @@ -1824,13 +2110,13 @@ static SRes LzmaEnc_CodeOneBlock(CLzmaEnc *p, Bool useLimits, UInt32 maxPackSize if (p->nowPos64 == 0) { - UInt32 numPairs; + unsigned numPairs; Byte curByte; if (p->matchFinder.GetNumAvailableBytes(p->matchFinderObj) == 0) return Flush(p, nowPos32); ReadMatchDistances(p, &numPairs); - RangeEnc_EncodeBit(&p->rc, &p->isMatch[p->state][0], 0); - p->state = kLiteralNextStates[p->state]; + RangeEnc_EncodeBit_0(&p->rc, &p->isMatch[kState_Start][0]); + // p->state = kLiteralNextStates[p->state]; curByte = *(p->matchFinder.GetPointerToCurrentPos(p->matchFinderObj) - p->additionalOffset); LitEnc_Encode(&p->rc, p->litProbs, curByte); p->additionalOffset--; @@ -1838,109 +2124,225 @@ static SRes LzmaEnc_CodeOneBlock(CLzmaEnc *p, Bool useLimits, UInt32 maxPackSize } if (p->matchFinder.GetNumAvailableBytes(p->matchFinderObj) != 0) + for (;;) { - UInt32 pos, len, posState; - + UInt32 dist; + unsigned len, posState; + UInt32 range, ttt, newBound; + CLzmaProb *probs; + if (p->fastMode) - len = GetOptimumFast(p, &pos); + len = GetOptimumFast(p); else - len = GetOptimum(p, nowPos32, &pos); + { + unsigned oci = p->optCur; + if (p->optEnd == oci) + len = GetOptimum(p, nowPos32); + else + { + const COptimal *opt = &p->opt[oci]; + len = opt->len; + p->backRes = opt->dist; + p->optCur = oci + 1; + } + } + + posState = (unsigned)nowPos32 & p->pbMask; + range = p->rc.range; + probs = &p->isMatch[p->state][posState]; + + RC_BIT_PRE(&p->rc, probs) + + dist = p->backRes; #ifdef SHOW_STAT2 - printf("\n pos = %4X, len = %u pos = %u", nowPos32, len, pos); + printf("\n pos = %6X, len = %3u pos = %6u", nowPos32, len, dist); #endif - posState = nowPos32 & p->pbMask; - if (len == 1 && pos == (UInt32)-1) + if (dist == MARK_LIT) { Byte curByte; - CLzmaProb *probs; const Byte *data; + unsigned state; - RangeEnc_EncodeBit(&p->rc, &p->isMatch[p->state][posState], 0); + RC_BIT_0(&p->rc, probs); + p->rc.range = range; data = p->matchFinder.GetPointerToCurrentPos(p->matchFinderObj) - p->additionalOffset; - curByte = *data; probs = LIT_PROBS(nowPos32, *(data - 1)); - if (IsCharState(p->state)) + curByte = *data; + state = p->state; + p->state = kLiteralNextStates[state]; + if (IsLitState(state)) LitEnc_Encode(&p->rc, probs, curByte); else - LitEnc_EncodeMatched(&p->rc, probs, curByte, *(data - p->reps[0] - 1)); - p->state = kLiteralNextStates[p->state]; + LitEnc_EncodeMatched(&p->rc, probs, curByte, *(data - p->reps[0])); } else { - RangeEnc_EncodeBit(&p->rc, &p->isMatch[p->state][posState], 1); - if (pos < LZMA_NUM_REPS) + RC_BIT_1(&p->rc, probs); + probs = &p->isRep[p->state]; + RC_BIT_PRE(&p->rc, probs) + + if (dist < LZMA_NUM_REPS) { - RangeEnc_EncodeBit(&p->rc, &p->isRep[p->state], 1); - if (pos == 0) + RC_BIT_1(&p->rc, probs); + probs = &p->isRepG0[p->state]; + RC_BIT_PRE(&p->rc, probs) + if (dist == 0) { - RangeEnc_EncodeBit(&p->rc, &p->isRepG0[p->state], 0); - RangeEnc_EncodeBit(&p->rc, &p->isRep0Long[p->state][posState], ((len == 1) ? 0 : 1)); + RC_BIT_0(&p->rc, probs); + probs = &p->isRep0Long[p->state][posState]; + RC_BIT_PRE(&p->rc, probs) + if (len != 1) + { + RC_BIT_1_BASE(&p->rc, probs); + } + else + { + RC_BIT_0_BASE(&p->rc, probs); + p->state = kShortRepNextStates[p->state]; + } } else { - UInt32 distance = p->reps[pos]; - RangeEnc_EncodeBit(&p->rc, &p->isRepG0[p->state], 1); - if (pos == 1) - RangeEnc_EncodeBit(&p->rc, &p->isRepG1[p->state], 0); + RC_BIT_1(&p->rc, probs); + probs = &p->isRepG1[p->state]; + RC_BIT_PRE(&p->rc, probs) + if (dist == 1) + { + RC_BIT_0_BASE(&p->rc, probs); + dist = p->reps[1]; + } else { - RangeEnc_EncodeBit(&p->rc, &p->isRepG1[p->state], 1); - RangeEnc_EncodeBit(&p->rc, &p->isRepG2[p->state], pos - 2); - if (pos == 3) + RC_BIT_1(&p->rc, probs); + probs = &p->isRepG2[p->state]; + RC_BIT_PRE(&p->rc, probs) + if (dist == 2) + { + RC_BIT_0_BASE(&p->rc, probs); + dist = p->reps[2]; + } + else + { + RC_BIT_1_BASE(&p->rc, probs); + dist = p->reps[3]; p->reps[3] = p->reps[2]; + } p->reps[2] = p->reps[1]; } p->reps[1] = p->reps[0]; - p->reps[0] = distance; + p->reps[0] = dist; } - if (len == 1) - p->state = kShortRepNextStates[p->state]; - else + + RC_NORM(&p->rc) + + p->rc.range = range; + + if (len != 1) { - LenEnc_Encode2(&p->repLenEnc, &p->rc, len - LZMA_MATCH_LEN_MIN, posState, !p->fastMode, p->ProbPrices); + LenEnc_Encode(&p->repLenProbs, &p->rc, len - LZMA_MATCH_LEN_MIN, posState); + if (!p->fastMode) + if (--p->repLenEnc.counters[posState] == 0) + LenPriceEnc_UpdateTable(&p->repLenEnc, posState, &p->repLenProbs, p->ProbPrices); + p->state = kRepNextStates[p->state]; } } else { - UInt32 posSlot; - RangeEnc_EncodeBit(&p->rc, &p->isRep[p->state], 0); + unsigned posSlot; + RC_BIT_0(&p->rc, probs); + p->rc.range = range; p->state = kMatchNextStates[p->state]; - LenEnc_Encode2(&p->lenEnc, &p->rc, len - LZMA_MATCH_LEN_MIN, posState, !p->fastMode, p->ProbPrices); - pos -= LZMA_NUM_REPS; - GetPosSlot(pos, posSlot); - RcTree_Encode(&p->rc, p->posSlotEncoder[GetLenToPosState(len)], kNumPosSlotBits, posSlot); - - if (posSlot >= kStartPosModelIndex) - { - UInt32 footerBits = ((posSlot >> 1) - 1); - UInt32 base = ((2 | (posSlot & 1)) << footerBits); - UInt32 posReduced = pos - base; - if (posSlot < kEndPosModelIndex) - RcTree_ReverseEncode(&p->rc, p->posEncoders + base - posSlot - 1, footerBits, posReduced); - else - { - RangeEnc_EncodeDirectBits(&p->rc, posReduced >> kNumAlignBits, footerBits - kNumAlignBits); - RcTree_ReverseEncode(&p->rc, p->posAlignEncoder, kNumAlignBits, posReduced & kAlignMask); - p->alignPriceCount++; - } - } + LenEnc_Encode(&p->lenProbs, &p->rc, len - LZMA_MATCH_LEN_MIN, posState); + if (!p->fastMode) + if (--p->lenEnc.counters[posState] == 0) + LenPriceEnc_UpdateTable(&p->lenEnc, posState, &p->lenProbs, p->ProbPrices); + + dist -= LZMA_NUM_REPS; p->reps[3] = p->reps[2]; p->reps[2] = p->reps[1]; p->reps[1] = p->reps[0]; - p->reps[0] = pos; + p->reps[0] = dist + 1; + p->matchPriceCount++; + GetPosSlot(dist, posSlot); + // RcTree_Encode_PosSlot(&p->rc, p->posSlotEncoder[GetLenToPosState(len)], posSlot); + { + UInt32 symbol = posSlot + (1 << kNumPosSlotBits); + range = p->rc.range; + probs = p->posSlotEncoder[GetLenToPosState(len)]; + do + { + CLzmaProb *prob = probs + (symbol >> kNumPosSlotBits); + UInt32 bit = (symbol >> (kNumPosSlotBits - 1)) & 1; + symbol <<= 1; + RC_BIT(&p->rc, prob, bit); + } + while (symbol < (1 << kNumPosSlotBits * 2)); + p->rc.range = range; + } + + if (dist >= kStartPosModelIndex) + { + unsigned footerBits = ((posSlot >> 1) - 1); + + if (dist < kNumFullDistances) + { + unsigned base = ((2 | (posSlot & 1)) << footerBits); + RcTree_ReverseEncode(&p->rc, p->posEncoders + base, footerBits, dist - base); + } + else + { + UInt32 pos2 = (dist | 0xF) << (32 - footerBits); + range = p->rc.range; + // RangeEnc_EncodeDirectBits(&p->rc, posReduced >> kNumAlignBits, footerBits - kNumAlignBits); + /* + do + { + range >>= 1; + p->rc.low += range & (0 - ((dist >> --footerBits) & 1)); + RC_NORM(&p->rc) + } + while (footerBits > kNumAlignBits); + */ + do + { + range >>= 1; + p->rc.low += range & (0 - (pos2 >> 31)); + pos2 += pos2; + RC_NORM(&p->rc) + } + while (pos2 != 0xF0000000); + + + // RcTree_ReverseEncode(&p->rc, p->posAlignEncoder, kNumAlignBits, posReduced & kAlignMask); + + { + unsigned m = 1; + unsigned bit; + bit = dist & 1; dist >>= 1; RC_BIT(&p->rc, p->posAlignEncoder + m, bit); m = (m << 1) + bit; + bit = dist & 1; dist >>= 1; RC_BIT(&p->rc, p->posAlignEncoder + m, bit); m = (m << 1) + bit; + bit = dist & 1; dist >>= 1; RC_BIT(&p->rc, p->posAlignEncoder + m, bit); m = (m << 1) + bit; + bit = dist & 1; RC_BIT(&p->rc, p->posAlignEncoder + m, bit); + p->rc.range = range; + p->alignPriceCount++; + } + } + } } } - p->additionalOffset -= len; + nowPos32 += len; + p->additionalOffset -= len; + if (p->additionalOffset == 0) { UInt32 processed; + if (!p->fastMode) { if (p->matchPriceCount >= (1 << 7)) @@ -1948,13 +2350,15 @@ static SRes LzmaEnc_CodeOneBlock(CLzmaEnc *p, Bool useLimits, UInt32 maxPackSize if (p->alignPriceCount >= kAlignTableSize) FillAlignPrices(p); } + if (p->matchFinder.GetNumAvailableBytes(p->matchFinderObj) == 0) break; processed = nowPos32 - startPos32; - if (useLimits) + + if (maxPackSize) { - if (processed + kNumOpts + 300 >= maxUnpackSize || - RangeEnc_GetProcessed(&p->rc) + kNumOpts * 2 >= maxPackSize) + if (processed + kNumOpts + 300 >= maxUnpackSize + || RangeEnc_GetProcessed_sizet(&p->rc) + kPackReserve >= maxPackSize) break; } else if (processed >= (1 << 17)) @@ -1964,10 +2368,13 @@ static SRes LzmaEnc_CodeOneBlock(CLzmaEnc *p, Bool useLimits, UInt32 maxPackSize } } } + p->nowPos64 += nowPos32 - startPos32; return Flush(p, nowPos32); } + + #define kBigHashDicLimit ((UInt32)1 << 24) static SRes LzmaEnc_Alloc(CLzmaEnc *p, UInt32 keepWindowSize, ISzAllocPtr alloc, ISzAllocPtr allocBig) @@ -2004,7 +2411,10 @@ static SRes LzmaEnc_Alloc(CLzmaEnc *p, UInt32 keepWindowSize, ISzAllocPtr alloc, #ifndef _7ZIP_ST if (p->mtMode) { - RINOK(MatchFinderMt_Create(&p->matchFinderMt, p->dictSize, beforeSize, p->numFastBytes, LZMA_MATCH_LEN_MAX, allocBig)); + RINOK(MatchFinderMt_Create(&p->matchFinderMt, p->dictSize, beforeSize, p->numFastBytes, + LZMA_MATCH_LEN_MAX + + 1 /* 18.04 */ + , allocBig)); p->matchFinderObj = &p->matchFinderMt; p->matchFinderBase.bigHash = (Byte)( (p->dictSize > kBigHashDicLimit && p->matchFinderBase.hashMask >= 0xFFFFFF) ? 1 : 0); @@ -2024,17 +2434,21 @@ static SRes LzmaEnc_Alloc(CLzmaEnc *p, UInt32 keepWindowSize, ISzAllocPtr alloc, void LzmaEnc_Init(CLzmaEnc *p) { - UInt32 i; + unsigned i; p->state = 0; - for (i = 0 ; i < LZMA_NUM_REPS; i++) - p->reps[i] = 0; + p->reps[0] = + p->reps[1] = + p->reps[2] = + p->reps[3] = 1; RangeEnc_Init(&p->rc); + for (i = 0; i < (1 << kNumAlignBits); i++) + p->posAlignEncoder[i] = kProbInitValue; for (i = 0; i < kNumStates; i++) { - UInt32 j; + unsigned j; for (j = 0; j < LZMA_NUM_PB_STATES_MAX; j++) { p->isMatch[i][j] = kProbInitValue; @@ -2046,39 +2460,38 @@ void LzmaEnc_Init(CLzmaEnc *p) p->isRepG2[i] = kProbInitValue; } - { - UInt32 num = (UInt32)0x300 << (p->lp + p->lc); - CLzmaProb *probs = p->litProbs; - for (i = 0; i < num; i++) - probs[i] = kProbInitValue; - } - { for (i = 0; i < kNumLenToPosStates; i++) { CLzmaProb *probs = p->posSlotEncoder[i]; - UInt32 j; + unsigned j; for (j = 0; j < (1 << kNumPosSlotBits); j++) probs[j] = kProbInitValue; } } { - for (i = 0; i < kNumFullDistances - kEndPosModelIndex; i++) + for (i = 0; i < kNumFullDistances; i++) p->posEncoders[i] = kProbInitValue; } - LenEnc_Init(&p->lenEnc.p); - LenEnc_Init(&p->repLenEnc.p); + { + UInt32 num = (UInt32)0x300 << (p->lp + p->lc); + UInt32 k; + CLzmaProb *probs = p->litProbs; + for (k = 0; k < num; k++) + probs[k] = kProbInitValue; + } - for (i = 0; i < (1 << kNumAlignBits); i++) - p->posAlignEncoder[i] = kProbInitValue; - p->optimumEndIndex = 0; - p->optimumCurrentIndex = 0; + LenEnc_Init(&p->lenProbs); + LenEnc_Init(&p->repLenProbs); + + p->optEnd = 0; + p->optCur = 0; p->additionalOffset = 0; p->pbMask = (1 << p->pb) - 1; - p->lpMask = (1 << p->lp) - 1; + p->lpMask = ((UInt32)0x100 << p->lp) - ((unsigned)0x100 >> p->lc); } void LzmaEnc_InitPrices(CLzmaEnc *p) @@ -2092,14 +2505,14 @@ void LzmaEnc_InitPrices(CLzmaEnc *p) p->lenEnc.tableSize = p->repLenEnc.tableSize = p->numFastBytes + 1 - LZMA_MATCH_LEN_MIN; - LenPriceEnc_UpdateTables(&p->lenEnc, 1 << p->pb, p->ProbPrices); - LenPriceEnc_UpdateTables(&p->repLenEnc, 1 << p->pb, p->ProbPrices); + LenPriceEnc_UpdateTables(&p->lenEnc, 1 << p->pb, &p->lenProbs, p->ProbPrices); + LenPriceEnc_UpdateTables(&p->repLenEnc, 1 << p->pb, &p->repLenProbs, p->ProbPrices); } static SRes LzmaEnc_AllocAndInit(CLzmaEnc *p, UInt32 keepWindowSize, ISzAllocPtr alloc, ISzAllocPtr allocBig) { - UInt32 i; - for (i = 0; i < (UInt32)kDicLogSizeMaxCompress; i++) + unsigned i; + for (i = kEndPosModelIndex / 2; i < kDicLogSizeMax; i++) if (p->dictSize <= ((UInt32)1 << i)) break; p->distTableSize = i * 2; @@ -2220,11 +2633,15 @@ SRes LzmaEnc_CodeOneMemBlock(CLzmaEncHandle pp, Bool reInit, if (reInit) LzmaEnc_Init(p); LzmaEnc_InitPrices(p); + nowPos64 = p->nowPos64; RangeEnc_Init(&p->rc); p->rc.outStream = &outStream.vt; - res = LzmaEnc_CodeOneBlock(p, True, desiredPackSize, *unpackSize); + if (desiredPackSize == 0) + return SZ_ERROR_OUTPUT_EOF; + + res = LzmaEnc_CodeOneBlock(p, desiredPackSize, *unpackSize); *unpackSize = (UInt32)(p->nowPos64 - nowPos64); *destLen -= outStream.rem; @@ -2247,7 +2664,7 @@ static SRes LzmaEnc_Encode2(CLzmaEnc *p, ICompressProgress *progress) for (;;) { - res = LzmaEnc_CodeOneBlock(p, False, 0, 0); + res = LzmaEnc_CodeOneBlock(p, 0, 0); if (res != SZ_OK || p->finished) break; if (progress) diff --git a/C/MtCoder.c b/C/MtCoder.c index 9cf53007..ddc7c028 100644 --- a/C/MtCoder.c +++ b/C/MtCoder.c @@ -1,85 +1,28 @@ /* MtCoder.c -- Multi-thread Coder -2017-07-17 : Igor Pavlov : Public domain */ +2018-02-21 : Igor Pavlov : Public domain */ #include "Precomp.h" #include "MtCoder.h" -static void MtProgress_Init(CMtProgress *p, ICompressProgress *progress) -{ - unsigned i; +#ifndef _7ZIP_ST - p->progress = progress; - p->res = SZ_OK; - p->totalInSize = 0; - p->totalOutSize = 0; - - for (i = 0; i < MTCODER__THREADS_MAX; i++) +SRes MtProgressThunk_Progress(const ICompressProgress *pp, UInt64 inSize, UInt64 outSize) +{ + CMtProgressThunk *thunk = CONTAINER_FROM_VTBL(pp, CMtProgressThunk, vt); + UInt64 inSize2 = 0; + UInt64 outSize2 = 0; + if (inSize != (UInt64)(Int64)-1) { - CMtProgressSizes *pair = &p->sizes[i]; - pair->inSize = 0; - pair->outSize = 0; + inSize2 = inSize - thunk->inSize; + thunk->inSize = inSize; } -} - - -static void MtProgress_Reinit(CMtProgress *p, unsigned index) -{ - CMtProgressSizes *pair = &p->sizes[index]; - pair->inSize = 0; - pair->outSize = 0; -} - - -#define UPDATE_PROGRESS(size, prev, total) \ - if (size != (UInt64)(Int64)-1) { total += size - prev; prev = size; } - -SRes MtProgress_Set(CMtProgress *p, unsigned index, UInt64 inSize, UInt64 outSize) -{ - SRes res; - CMtProgressSizes *pair; - - CriticalSection_Enter(&p->cs); - - pair = &p->sizes[index]; - UPDATE_PROGRESS(inSize, pair->inSize, p->totalInSize) - UPDATE_PROGRESS(outSize, pair->outSize, p->totalOutSize) - if (p->res == SZ_OK && p->progress) + if (outSize != (UInt64)(Int64)-1) { - if (ICompressProgress_Progress(p->progress, p->totalInSize, p->totalOutSize) != SZ_OK) - p->res = SZ_ERROR_PROGRESS; + outSize2 = outSize - thunk->outSize; + thunk->outSize = outSize; } - res = p->res; - - CriticalSection_Leave(&p->cs); - - return res; -} - - -static SRes MtProgress_GetError(CMtProgress *p) -{ - SRes res; - CriticalSection_Enter(&p->cs); - res = p->res; - CriticalSection_Leave(&p->cs); - return res; -} - - -static void MtProgress_SetError(CMtProgress *p, SRes res) -{ - CriticalSection_Enter(&p->cs); - if (p->res == SZ_OK) - p->res = res; - CriticalSection_Leave(&p->cs); -} - - -static SRes MtProgressThunk_Progress(const ICompressProgress *pp, UInt64 inSize, UInt64 outSize) -{ - CMtProgressThunk *p = CONTAINER_FROM_VTBL(pp, CMtProgressThunk, vt); - return MtProgress_Set(p->mtProgress, p->index, inSize, outSize); + return MtProgress_ProgressAdd(thunk->mtProgress, inSize2, outSize2); } @@ -298,7 +241,7 @@ static SRes ThreadFunc2(CMtCoderThread *t) res = mtc->mtCallback->Code(mtc->mtCallbackObject, t->index, bufIndex, mtc->inStream ? t->inBuf : inData, size, finished); - MtProgress_Reinit(&mtc->mtProgress, t->index); + // MtProgress_Reinit(&mtc->mtProgress, t->index); if (res != SZ_OK) MtProgress_SetError(&mtc->mtProgress, res); @@ -654,3 +597,5 @@ SRes MtCoder_Code(CMtCoder *p) MtCoder_Free(p); return res; } + +#endif diff --git a/C/MtCoder.h b/C/MtCoder.h index 46674a31..7982e847 100644 --- a/C/MtCoder.h +++ b/C/MtCoder.h @@ -1,10 +1,10 @@ /* MtCoder.h -- Multi-thread Coder -2017-06-18 : Igor Pavlov : Public domain */ +2018-02-21 : Igor Pavlov : Public domain */ #ifndef __MT_CODER_H #define __MT_CODER_H -#include "Threads.h" +#include "MtDec.h" EXTERN_C_BEGIN @@ -24,33 +24,20 @@ EXTERN_C_BEGIN #endif -typedef struct -{ - UInt64 inSize; - UInt64 outSize; -} CMtProgressSizes; - - -typedef struct -{ - ICompressProgress *progress; - SRes res; - UInt64 totalInSize; - UInt64 totalOutSize; - CCriticalSection cs; - CMtProgressSizes sizes[MTCODER__THREADS_MAX]; -} CMtProgress; +#ifndef _7ZIP_ST typedef struct { ICompressProgress vt; CMtProgress *mtProgress; - unsigned index; + UInt64 inSize; + UInt64 outSize; } CMtProgressThunk; void MtProgressThunk_CreateVTable(CMtProgressThunk *p); +#define MtProgressThunk_Init(p) { (p)->inSize = 0; (p)->outSize = 0; } struct _CMtCoder; @@ -146,6 +133,9 @@ void MtCoder_Destruct(CMtCoder *p); SRes MtCoder_Code(CMtCoder *p); +#endif + + EXTERN_C_END #endif diff --git a/C/MtDec.c b/C/MtDec.c new file mode 100644 index 00000000..60d31b07 --- /dev/null +++ b/C/MtDec.c @@ -0,0 +1,1137 @@ +/* MtDec.c -- Multi-thread Decoder +2018-03-02 : Igor Pavlov : Public domain */ + +#include "Precomp.h" + +// #define SHOW_DEBUG_INFO + +// #include + +#ifdef SHOW_DEBUG_INFO +#include +#endif + +#ifdef SHOW_DEBUG_INFO +#define PRF(x) x +#else +#define PRF(x) +#endif + +#define PRF_STR_INT(s, d) PRF(printf("\n" s " %d\n", (unsigned)d)) + +#include "MtDec.h" + +#ifndef _7ZIP_ST + +void MtProgress_Init(CMtProgress *p, ICompressProgress *progress) +{ + p->progress = progress; + p->res = SZ_OK; + p->totalInSize = 0; + p->totalOutSize = 0; +} + + +SRes MtProgress_Progress_ST(CMtProgress *p) +{ + if (p->res == SZ_OK && p->progress) + if (ICompressProgress_Progress(p->progress, p->totalInSize, p->totalOutSize) != SZ_OK) + p->res = SZ_ERROR_PROGRESS; + return p->res; +} + + +SRes MtProgress_ProgressAdd(CMtProgress *p, UInt64 inSize, UInt64 outSize) +{ + SRes res; + CriticalSection_Enter(&p->cs); + + p->totalInSize += inSize; + p->totalOutSize += outSize; + if (p->res == SZ_OK && p->progress) + if (ICompressProgress_Progress(p->progress, p->totalInSize, p->totalOutSize) != SZ_OK) + p->res = SZ_ERROR_PROGRESS; + res = p->res; + + CriticalSection_Leave(&p->cs); + return res; +} + + +SRes MtProgress_GetError(CMtProgress *p) +{ + SRes res; + CriticalSection_Enter(&p->cs); + res = p->res; + CriticalSection_Leave(&p->cs); + return res; +} + + +void MtProgress_SetError(CMtProgress *p, SRes res) +{ + CriticalSection_Enter(&p->cs); + if (p->res == SZ_OK) + p->res = res; + CriticalSection_Leave(&p->cs); +} + + +#define RINOK_THREAD(x) RINOK(x) + + +static WRes ArEvent_OptCreate_And_Reset(CEvent *p) +{ + if (Event_IsCreated(p)) + return Event_Reset(p); + return AutoResetEvent_CreateNotSignaled(p); +} + + + +typedef struct +{ + void *next; + void *pad[3]; +} CMtDecBufLink; + +#define MTDEC__LINK_DATA_OFFSET sizeof(CMtDecBufLink) +#define MTDEC__DATA_PTR_FROM_LINK(link) ((Byte *)(link) + MTDEC__LINK_DATA_OFFSET) + + + +static THREAD_FUNC_RET_TYPE THREAD_FUNC_CALL_TYPE ThreadFunc(void *pp); + + +static WRes MtDecThread_CreateEvents(CMtDecThread *t) +{ + WRes wres = ArEvent_OptCreate_And_Reset(&t->canWrite); + if (wres == 0) + { + wres = ArEvent_OptCreate_And_Reset(&t->canRead); + if (wres == 0) + return SZ_OK; + } + return wres; +} + + +static SRes MtDecThread_CreateAndStart(CMtDecThread *t) +{ + WRes wres = MtDecThread_CreateEvents(t); + // wres = 17; // for test + if (wres == 0) + { + if (Thread_WasCreated(&t->thread)) + return SZ_OK; + wres = Thread_Create(&t->thread, ThreadFunc, t); + if (wres == 0) + return SZ_OK; + } + return MY_SRes_HRESULT_FROM_WRes(wres); +} + + +void MtDecThread_FreeInBufs(CMtDecThread *t) +{ + if (t->inBuf) + { + void *link = t->inBuf; + t->inBuf = NULL; + do + { + void *next = ((CMtDecBufLink *)link)->next; + ISzAlloc_Free(t->mtDec->alloc, link); + link = next; + } + while (link); + } +} + + +static void MtDecThread_CloseThread(CMtDecThread *t) +{ + if (Thread_WasCreated(&t->thread)) + { + Event_Set(&t->canWrite); /* we can disable it. There are no threads waiting canWrite in normal cases */ + Event_Set(&t->canRead); + Thread_Wait(&t->thread); + Thread_Close(&t->thread); + } + + Event_Close(&t->canRead); + Event_Close(&t->canWrite); +} + +static void MtDec_CloseThreads(CMtDec *p) +{ + unsigned i; + for (i = 0; i < MTDEC__THREADS_MAX; i++) + MtDecThread_CloseThread(&p->threads[i]); +} + +static void MtDecThread_Destruct(CMtDecThread *t) +{ + MtDecThread_CloseThread(t); + MtDecThread_FreeInBufs(t); +} + + + +static SRes FullRead(ISeqInStream *stream, Byte *data, size_t *processedSize) +{ + size_t size = *processedSize; + *processedSize = 0; + while (size != 0) + { + size_t cur = size; + SRes res = ISeqInStream_Read(stream, data, &cur); + *processedSize += cur; + data += cur; + size -= cur; + RINOK(res); + if (cur == 0) + return SZ_OK; + } + return SZ_OK; +} + + +static SRes MtDec_GetError_Spec(CMtDec *p, UInt64 interruptIndex, Bool *wasInterrupted) +{ + SRes res; + CriticalSection_Enter(&p->mtProgress.cs); + *wasInterrupted = (p->needInterrupt && interruptIndex > p->interruptIndex); + res = p->mtProgress.res; + CriticalSection_Leave(&p->mtProgress.cs); + return res; +} + +static SRes MtDec_Progress_GetError_Spec(CMtDec *p, UInt64 inSize, UInt64 outSize, UInt64 interruptIndex, Bool *wasInterrupted) +{ + SRes res; + CriticalSection_Enter(&p->mtProgress.cs); + + p->mtProgress.totalInSize += inSize; + p->mtProgress.totalOutSize += outSize; + if (p->mtProgress.res == SZ_OK && p->mtProgress.progress) + if (ICompressProgress_Progress(p->mtProgress.progress, p->mtProgress.totalInSize, p->mtProgress.totalOutSize) != SZ_OK) + p->mtProgress.res = SZ_ERROR_PROGRESS; + + *wasInterrupted = (p->needInterrupt && interruptIndex > p->interruptIndex); + res = p->mtProgress.res; + + CriticalSection_Leave(&p->mtProgress.cs); + + return res; +} + +static void MtDec_Interrupt(CMtDec *p, UInt64 interruptIndex) +{ + CriticalSection_Enter(&p->mtProgress.cs); + if (!p->needInterrupt || interruptIndex < p->interruptIndex) + { + p->interruptIndex = interruptIndex; + p->needInterrupt = True; + } + CriticalSection_Leave(&p->mtProgress.cs); +} + +Byte *MtDec_GetCrossBuff(CMtDec *p) +{ + Byte *cr = p->crossBlock; + if (!cr) + { + cr = (Byte *)ISzAlloc_Alloc(p->alloc, MTDEC__LINK_DATA_OFFSET + p->inBufSize); + if (!cr) + return NULL; + p->crossBlock = cr; + } + return MTDEC__DATA_PTR_FROM_LINK(cr); +} + + +/* + ThreadFunc2() returns: + 0 - in all normal cases (even for stream error or memory allocation error) + (!= 0) - WRes error return by system threading function +*/ + +// #define MTDEC_ProgessStep (1 << 22) +#define MTDEC_ProgessStep (1 << 0) + +static WRes ThreadFunc2(CMtDecThread *t) +{ + CMtDec *p = t->mtDec; + + PRF_STR_INT("ThreadFunc2", t->index); + + // SetThreadAffinityMask(GetCurrentThread(), 1 << t->index); + + for (;;) + { + SRes res, codeRes; + Bool wasInterrupted, isAllocError, overflow, finish; + SRes threadingErrorSRes; + Bool needCode, needWrite, needContinue; + + size_t inDataSize_Start; + UInt64 inDataSize; + // UInt64 inDataSize_Full; + + UInt64 blockIndex; + + UInt64 inPrev = 0; + UInt64 outPrev = 0; + UInt64 inCodePos; + UInt64 outCodePos; + + Byte *afterEndData = NULL; + size_t afterEndData_Size = 0; + + Bool canCreateNewThread = False; + // CMtDecCallbackInfo parse; + CMtDecThread *nextThread; + + PRF_STR_INT("Event_Wait(&t->canRead)", t->index); + + RINOK_THREAD(Event_Wait(&t->canRead)); + if (p->exitThread) + return 0; + + PRF_STR_INT("after Event_Wait(&t->canRead)", t->index); + + // if (t->index == 3) return 19; // for test + + blockIndex = p->blockIndex++; + + // PRF(printf("\ncanRead\n")) + + res = MtDec_Progress_GetError_Spec(p, 0, 0, blockIndex, &wasInterrupted); + + finish = p->readWasFinished; + needCode = False; + needWrite = False; + isAllocError = False; + overflow = False; + + inDataSize_Start = 0; + inDataSize = 0; + // inDataSize_Full = 0; + + if (res == SZ_OK && !wasInterrupted) + { + // if (p->inStream) + { + CMtDecBufLink *prev = NULL; + CMtDecBufLink *link = (CMtDecBufLink *)t->inBuf; + size_t crossSize = p->crossEnd - p->crossStart; + + PRF(printf("\ncrossSize = %d\n", crossSize)); + + for (;;) + { + if (!link) + { + link = (CMtDecBufLink *)ISzAlloc_Alloc(p->alloc, MTDEC__LINK_DATA_OFFSET + p->inBufSize); + if (!link) + { + finish = True; + // p->allocError_for_Read_BlockIndex = blockIndex; + isAllocError = True; + break; + } + link->next = NULL; + if (prev) + { + // static unsigned g_num = 0; + // printf("\n%6d : %x", ++g_num, (unsigned)(size_t)((Byte *)link - (Byte *)prev)); + prev->next = link; + } + else + t->inBuf = (void *)link; + } + + { + Byte *data = MTDEC__DATA_PTR_FROM_LINK(link); + Byte *parseData = data; + size_t size; + + if (crossSize != 0) + { + inDataSize = crossSize; + // inDataSize_Full = inDataSize; + inDataSize_Start = crossSize; + size = crossSize; + parseData = MTDEC__DATA_PTR_FROM_LINK(p->crossBlock) + p->crossStart; + PRF(printf("\ncross : crossStart = %7d crossEnd = %7d finish = %1d", + (int)p->crossStart, (int)p->crossEnd, (int)finish)); + } + else + { + size = p->inBufSize; + + res = FullRead(p->inStream, data, &size); + + // size = 10; // test + + inDataSize += size; + // inDataSize_Full = inDataSize; + if (!prev) + inDataSize_Start = size; + + p->readProcessed += size; + finish = (size != p->inBufSize); + if (finish) + p->readWasFinished = True; + + // res = E_INVALIDARG; // test + + if (res != SZ_OK) + { + // PRF(printf("\nRead error = %d\n", res)) + // we want to decode all data before error + p->readRes = res; + // p->readError_BlockIndex = blockIndex; + p->readWasFinished = True; + finish = True; + res = SZ_OK; + // break; + } + + if (inDataSize - inPrev >= MTDEC_ProgessStep) + { + res = MtDec_Progress_GetError_Spec(p, 0, 0, blockIndex, &wasInterrupted); + if (res != SZ_OK || wasInterrupted) + break; + inPrev = inDataSize; + } + } + + { + CMtDecCallbackInfo parse; + + parse.startCall = (prev == NULL); + parse.src = parseData; + parse.srcSize = size; + parse.srcFinished = finish; + parse.canCreateNewThread = True; + + // PRF(printf("\nParse size = %d\n", (unsigned)size)) + + p->mtCallback->Parse(p->mtCallbackObject, t->index, &parse); + + needWrite = True; + canCreateNewThread = parse.canCreateNewThread; + + // printf("\n\n%12I64u %12I64u", (UInt64)p->mtProgress.totalInSize, (UInt64)p->mtProgress.totalOutSize); + + if ( + // parseRes != SZ_OK || + // inDataSize - (size - parse.srcSize) > p->inBlockMax + // || + parse.state == MTDEC_PARSE_OVERFLOW + // || wasInterrupted + ) + { + // Overflow or Parse error - switch from MT decoding to ST decoding + finish = True; + overflow = True; + + { + PRF(printf("\n Overflow")); + // PRF(printf("\nisBlockFinished = %d", (unsigned)parse.blockWasFinished)); + PRF(printf("\n inDataSize = %d", (unsigned)inDataSize)); + } + + if (crossSize != 0) + memcpy(data, parseData, size); + p->crossStart = 0; + p->crossEnd = 0; + break; + } + + if (crossSize != 0) + { + memcpy(data, parseData, parse.srcSize); + p->crossStart += parse.srcSize; + } + + if (parse.state != MTDEC_PARSE_CONTINUE || finish) + { + // we don't need to parse in current thread anymore + + if (parse.state == MTDEC_PARSE_END) + finish = True; + + needCode = True; + // p->crossFinished = finish; + + if (parse.srcSize == size) + { + // full parsed - no cross transfer + p->crossStart = 0; + p->crossEnd = 0; + break; + } + + if (parse.state == MTDEC_PARSE_END) + { + p->crossStart = 0; + p->crossEnd = 0; + + if (crossSize != 0) + memcpy(data + parse.srcSize, parseData + parse.srcSize, size - parse.srcSize); // we need all data + afterEndData_Size = size - parse.srcSize; + afterEndData = parseData + parse.srcSize; + + // we reduce data size to required bytes (parsed only) + inDataSize -= (size - parse.srcSize); + if (!prev) + inDataSize_Start = parse.srcSize; + break; + } + + { + // partial parsed - need cross transfer + if (crossSize != 0) + inDataSize = parse.srcSize; // it's only parsed now + else + { + // partial parsed - is not in initial cross block - we need to copy new data to cross block + Byte *cr = MtDec_GetCrossBuff(p); + if (!cr) + { + { + PRF(printf("\ncross alloc error error\n")); + // res = SZ_ERROR_MEM; + finish = True; + // p->allocError_for_Read_BlockIndex = blockIndex; + isAllocError = True; + break; + } + } + + { + size_t crSize = size - parse.srcSize; + inDataSize -= crSize; + p->crossEnd = crSize; + p->crossStart = 0; + memcpy(cr, parseData + parse.srcSize, crSize); + } + } + + // inDataSize_Full = inDataSize; + if (!prev) + inDataSize_Start = parse.srcSize; // it's partial size (parsed only) + + finish = False; + break; + } + } + + if (parse.srcSize != size) + { + res = SZ_ERROR_FAIL; + PRF(printf("\nfinished error SZ_ERROR_FAIL = %d\n", res)); + break; + } + } + } + + prev = link; + link = link->next; + + if (crossSize != 0) + { + crossSize = 0; + p->crossStart = 0; + p->crossEnd = 0; + } + } + } + + if (res == SZ_OK) + res = MtDec_GetError_Spec(p, blockIndex, &wasInterrupted); + } + + codeRes = SZ_OK; + + if (res == SZ_OK && needCode && !wasInterrupted) + { + codeRes = p->mtCallback->PreCode(p->mtCallbackObject, t->index); + if (codeRes != SZ_OK) + { + needCode = False; + finish = True; + // SZ_ERROR_MEM is expected error here. + // if (codeRes == SZ_ERROR_MEM) - we will try single-thread decoding later. + // if (codeRes != SZ_ERROR_MEM) - we can stop decoding or try single-thread decoding. + } + } + + if (res != SZ_OK || wasInterrupted) + finish = True; + + nextThread = NULL; + threadingErrorSRes = SZ_OK; + + if (!finish) + { + if (p->numStartedThreads < p->numStartedThreads_Limit && canCreateNewThread) + { + SRes res2 = MtDecThread_CreateAndStart(&p->threads[p->numStartedThreads]); + if (res2 == SZ_OK) + { + // if (p->numStartedThreads % 1000 == 0) PRF(printf("\n numStartedThreads=%d\n", p->numStartedThreads)); + p->numStartedThreads++; + } + else + { + PRF(printf("\nERROR: numStartedThreads=%d\n", p->numStartedThreads)); + if (p->numStartedThreads == 1) + { + // if only one thread is possible, we leave muti-threading code + finish = True; + needCode = False; + threadingErrorSRes = res2; + } + else + p->numStartedThreads_Limit = p->numStartedThreads; + } + } + + if (!finish) + { + unsigned nextIndex = t->index + 1; + nextThread = &p->threads[nextIndex >= p->numStartedThreads ? 0 : nextIndex]; + RINOK_THREAD(Event_Set(&nextThread->canRead)) + // We have started executing for new iteration (with next thread) + // And that next thread now is responsible for possible exit from decoding (threading_code) + } + } + + // each call of Event_Set(&nextThread->canRead) must be followed by call of Event_Set(&nextThread->canWrite) + // if ( !finish ) we must call Event_Set(&nextThread->canWrite) in any case + // if ( finish ) we switch to single-thread mode and there are 2 ways at the end of current iteration (current block): + // - if (needContinue) after Write(&needContinue), we restore decoding with new iteration + // - otherwise we stop decoding and exit from ThreadFunc2() + + // Don't change (finish) variable in the further code + + + // ---------- CODE ---------- + + inPrev = 0; + outPrev = 0; + inCodePos = 0; + outCodePos = 0; + + if (res == SZ_OK && needCode && codeRes == SZ_OK) + { + Bool isStartBlock = True; + CMtDecBufLink *link = (CMtDecBufLink *)t->inBuf; + + for (;;) + { + size_t inSize; + int stop; + + if (isStartBlock) + inSize = inDataSize_Start; + else + { + UInt64 rem = inDataSize - inCodePos; + inSize = p->inBufSize; + if (inSize > rem) + inSize = (size_t)rem; + } + + inCodePos += inSize; + stop = True; + + codeRes = p->mtCallback->Code(p->mtCallbackObject, t->index, + (const Byte *)MTDEC__DATA_PTR_FROM_LINK(link), inSize, + (inCodePos == inDataSize), // srcFinished + &inCodePos, &outCodePos, &stop); + + if (codeRes != SZ_OK) + { + PRF(printf("\nCode Interrupt error = %x\n", codeRes)); + // we interrupt only later blocks + MtDec_Interrupt(p, blockIndex); + break; + } + + if (stop || inCodePos == inDataSize) + break; + + { + const UInt64 inDelta = inCodePos - inPrev; + const UInt64 outDelta = outCodePos - outPrev; + if (inDelta >= MTDEC_ProgessStep || outDelta >= MTDEC_ProgessStep) + { + // Sleep(1); + res = MtDec_Progress_GetError_Spec(p, inDelta, outDelta, blockIndex, &wasInterrupted); + if (res != SZ_OK || wasInterrupted) + break; + inPrev = inCodePos; + outPrev = outCodePos; + } + } + + link = link->next; + isStartBlock = False; + } + } + + + // ---------- WRITE ---------- + + RINOK_THREAD(Event_Wait(&t->canWrite)); + + { + Bool isErrorMode = False; + Bool canRecode = True; + Bool needWriteToStream = needWrite; + + if (p->exitThread) return 0; // it's never executed in normal cases + + if (p->wasInterrupted) + wasInterrupted = True; + else + { + if (codeRes != SZ_OK) // || !needCode // check it !!! + { + p->wasInterrupted = True; + p->codeRes = codeRes; + if (codeRes == SZ_ERROR_MEM) + isAllocError = True; + } + + if (threadingErrorSRes) + { + p->wasInterrupted = True; + p->threadingErrorSRes = threadingErrorSRes; + needWriteToStream = False; + } + if (isAllocError) + { + p->wasInterrupted = True; + p->isAllocError = True; + needWriteToStream = False; + } + if (overflow) + { + p->wasInterrupted = True; + p->overflow = True; + needWriteToStream = False; + } + } + + if (needCode) + { + if (wasInterrupted) + { + inCodePos = 0; + outCodePos = 0; + } + { + const UInt64 inDelta = inCodePos - inPrev; + const UInt64 outDelta = outCodePos - outPrev; + // if (inDelta != 0 || outDelta != 0) + res = MtProgress_ProgressAdd(&p->mtProgress, inDelta, outDelta); + } + } + + needContinue = (!finish); + + // if (res == SZ_OK && needWrite && !wasInterrupted) + if (needWrite) + { + // p->inProcessed += inCodePos; + + res = p->mtCallback->Write(p->mtCallbackObject, t->index, + res == SZ_OK && needWriteToStream && !wasInterrupted, // needWrite + afterEndData, afterEndData_Size, + &needContinue, + &canRecode); + + // res= E_INVALIDARG; // for test + + PRF(printf("\nAfter Write needContinue = %d\n", (unsigned)needContinue)); + PRF(printf("\nprocessed = %d\n", (unsigned)p->inProcessed)); + + if (res != SZ_OK) + { + PRF(printf("\nWrite error = %d\n", res)); + isErrorMode = True; + p->wasInterrupted = True; + } + if (res != SZ_OK + || (!needContinue && !finish)) + { + PRF(printf("\nWrite Interrupt error = %x\n", res)); + MtDec_Interrupt(p, blockIndex); + } + } + + if (canRecode) + if (!needCode + || res != SZ_OK + || p->wasInterrupted + || codeRes != SZ_OK + || wasInterrupted + || p->numFilledThreads != 0 + || isErrorMode) + { + if (p->numFilledThreads == 0) + p->filledThreadStart = t->index; + if (inDataSize != 0 || !finish) + { + t->inDataSize_Start = inDataSize_Start; + t->inDataSize = inDataSize; + p->numFilledThreads++; + } + PRF(printf("\np->numFilledThreads = %d\n", p->numFilledThreads)); + PRF(printf("p->filledThreadStart = %d\n", p->filledThreadStart)); + } + + if (!finish) + { + RINOK_THREAD(Event_Set(&nextThread->canWrite)); + } + else + { + if (needContinue) + { + // we restore decoding with new iteration + RINOK_THREAD(Event_Set(&p->threads[0].canWrite)); + } + else + { + // we exit from decoding + if (t->index == 0) + return SZ_OK; + p->exitThread = True; + } + RINOK_THREAD(Event_Set(&p->threads[0].canRead)); + } + } + } +} + +#ifdef _WIN32 +#define USE_ALLOCA +#endif + +#ifdef USE_ALLOCA +#ifdef _WIN32 +#include +#else +#include +#endif +#endif + + +static THREAD_FUNC_RET_TYPE THREAD_FUNC_CALL_TYPE ThreadFunc1(void *pp) +{ + WRes res; + + CMtDecThread *t = (CMtDecThread *)pp; + CMtDec *p; + + // fprintf(stdout, "\n%d = %p\n", t->index, &t); + + res = ThreadFunc2(t); + p = t->mtDec; + if (res == 0) + return p->exitThreadWRes; + { + // it's unexpected situation for some threading function error + if (p->exitThreadWRes == 0) + p->exitThreadWRes = res; + PRF(printf("\nthread exit error = %d\n", res)); + p->exitThread = True; + Event_Set(&p->threads[0].canRead); + Event_Set(&p->threads[0].canWrite); + MtProgress_SetError(&p->mtProgress, MY_SRes_HRESULT_FROM_WRes(res)); + } + return res; +} + +static MY_NO_INLINE THREAD_FUNC_RET_TYPE THREAD_FUNC_CALL_TYPE ThreadFunc(void *pp) +{ + CMtDecThread *t = (CMtDecThread *)pp; + + // fprintf(stderr, "\n%d = %p - before", t->index, &t); + #ifdef USE_ALLOCA + t->allocaPtr = alloca(t->index * 128); + #endif + return ThreadFunc1(pp); +} + + +int MtDec_PrepareRead(CMtDec *p) +{ + if (p->crossBlock && p->crossStart == p->crossEnd) + { + ISzAlloc_Free(p->alloc, p->crossBlock); + p->crossBlock = NULL; + } + + { + unsigned i; + for (i = 0; i < MTDEC__THREADS_MAX; i++) + if (i > p->numStartedThreads + || p->numFilledThreads <= + (i >= p->filledThreadStart ? + i - p->filledThreadStart : + i + p->numStartedThreads - p->filledThreadStart)) + MtDecThread_FreeInBufs(&p->threads[i]); + } + + return (p->numFilledThreads != 0) || (p->crossStart != p->crossEnd); +} + + +const Byte *MtDec_Read(CMtDec *p, size_t *inLim) +{ + while (p->numFilledThreads != 0) + { + CMtDecThread *t = &p->threads[p->filledThreadStart]; + + if (*inLim != 0) + { + { + void *link = t->inBuf; + void *next = ((CMtDecBufLink *)link)->next; + ISzAlloc_Free(p->alloc, link); + t->inBuf = next; + } + + if (t->inDataSize == 0) + { + MtDecThread_FreeInBufs(t); + if (--p->numFilledThreads == 0) + break; + if (++p->filledThreadStart == p->numStartedThreads) + p->filledThreadStart = 0; + t = &p->threads[p->filledThreadStart]; + } + } + + { + size_t lim = t->inDataSize_Start; + if (lim != 0) + t->inDataSize_Start = 0; + else + { + UInt64 rem = t->inDataSize; + lim = p->inBufSize; + if (lim > rem) + lim = (size_t)rem; + } + t->inDataSize -= lim; + *inLim = lim; + return (const Byte *)MTDEC__DATA_PTR_FROM_LINK(t->inBuf); + } + } + + { + size_t crossSize = p->crossEnd - p->crossStart; + if (crossSize != 0) + { + const Byte *data = MTDEC__DATA_PTR_FROM_LINK(p->crossBlock) + p->crossStart; + *inLim = crossSize; + p->crossStart = 0; + p->crossEnd = 0; + return data; + } + *inLim = 0; + if (p->crossBlock) + { + ISzAlloc_Free(p->alloc, p->crossBlock); + p->crossBlock = NULL; + } + return NULL; + } +} + + +void MtDec_Construct(CMtDec *p) +{ + unsigned i; + + p->inBufSize = (size_t)1 << 18; + + p->numThreadsMax = 0; + + p->inStream = NULL; + + // p->inData = NULL; + // p->inDataSize = 0; + + p->crossBlock = NULL; + p->crossStart = 0; + p->crossEnd = 0; + + p->numFilledThreads = 0; + + p->progress = NULL; + p->alloc = NULL; + + p->mtCallback = NULL; + p->mtCallbackObject = NULL; + + p->allocatedBufsSize = 0; + + for (i = 0; i < MTDEC__THREADS_MAX; i++) + { + CMtDecThread *t = &p->threads[i]; + t->mtDec = p; + t->index = i; + t->inBuf = NULL; + Event_Construct(&t->canRead); + Event_Construct(&t->canWrite); + Thread_Construct(&t->thread); + } + + // Event_Construct(&p->finishedEvent); + + CriticalSection_Init(&p->mtProgress.cs); +} + + +static void MtDec_Free(CMtDec *p) +{ + unsigned i; + + p->exitThread = True; + + for (i = 0; i < MTDEC__THREADS_MAX; i++) + MtDecThread_Destruct(&p->threads[i]); + + // Event_Close(&p->finishedEvent); + + if (p->crossBlock) + { + ISzAlloc_Free(p->alloc, p->crossBlock); + p->crossBlock = NULL; + } +} + + +void MtDec_Destruct(CMtDec *p) +{ + MtDec_Free(p); + + CriticalSection_Delete(&p->mtProgress.cs); +} + + +SRes MtDec_Code(CMtDec *p) +{ + unsigned i; + + p->inProcessed = 0; + + p->blockIndex = 1; // it must be larger than not_defined index (0) + p->isAllocError = False; + p->overflow = False; + p->threadingErrorSRes = SZ_OK; + + p->needContinue = True; + + p->readWasFinished = False; + p->needInterrupt = False; + p->interruptIndex = (UInt64)(Int64)-1; + + p->readProcessed = 0; + p->readRes = SZ_OK; + p->codeRes = SZ_OK; + p->wasInterrupted = False; + + p->crossStart = 0; + p->crossEnd = 0; + + p->filledThreadStart = 0; + p->numFilledThreads = 0; + + { + unsigned numThreads = p->numThreadsMax; + if (numThreads > MTDEC__THREADS_MAX) + numThreads = MTDEC__THREADS_MAX; + p->numStartedThreads_Limit = numThreads; + p->numStartedThreads = 0; + } + + if (p->inBufSize != p->allocatedBufsSize) + { + for (i = 0; i < MTDEC__THREADS_MAX; i++) + { + CMtDecThread *t = &p->threads[i]; + if (t->inBuf) + MtDecThread_FreeInBufs(t); + } + if (p->crossBlock) + { + ISzAlloc_Free(p->alloc, p->crossBlock); + p->crossBlock = NULL; + } + + p->allocatedBufsSize = p->inBufSize; + } + + MtProgress_Init(&p->mtProgress, p->progress); + + // RINOK_THREAD(ArEvent_OptCreate_And_Reset(&p->finishedEvent)); + p->exitThread = False; + p->exitThreadWRes = 0; + + { + WRes wres; + WRes sres; + CMtDecThread *nextThread = &p->threads[p->numStartedThreads++]; + // wres = MtDecThread_CreateAndStart(nextThread); + wres = MtDecThread_CreateEvents(nextThread); + if (wres == 0) { wres = Event_Set(&nextThread->canWrite); + if (wres == 0) { wres = Event_Set(&nextThread->canRead); + if (wres == 0) { wres = ThreadFunc(nextThread); + if (wres != 0) + { + p->needContinue = False; + MtDec_CloseThreads(p); + }}}} + + // wres = 17; // for test + // wres = Event_Wait(&p->finishedEvent); + + sres = MY_SRes_HRESULT_FROM_WRes(wres); + + if (sres != 0) + p->threadingErrorSRes = sres; + + if ( + // wres == 0 + // wres != 0 + // || p->mtc.codeRes == SZ_ERROR_MEM + p->isAllocError + || p->threadingErrorSRes != SZ_OK + || p->overflow) + { + // p->needContinue = True; + } + else + p->needContinue = False; + + if (p->needContinue) + return SZ_OK; + + // if (sres != SZ_OK) + return sres; + // return E_FAIL; + } +} + +#endif diff --git a/C/MtDec.h b/C/MtDec.h new file mode 100644 index 00000000..b445bc9d --- /dev/null +++ b/C/MtDec.h @@ -0,0 +1,201 @@ +/* MtDec.h -- Multi-thread Decoder +2018-03-02 : Igor Pavlov : Public domain */ + +#ifndef __MT_DEC_H +#define __MT_DEC_H + +#include "7zTypes.h" + +#ifndef _7ZIP_ST +#include "Threads.h" +#endif + +EXTERN_C_BEGIN + +#ifndef _7ZIP_ST + +#ifndef _7ZIP_ST + #define MTDEC__THREADS_MAX 32 +#else + #define MTDEC__THREADS_MAX 1 +#endif + + +typedef struct +{ + ICompressProgress *progress; + SRes res; + UInt64 totalInSize; + UInt64 totalOutSize; + CCriticalSection cs; +} CMtProgress; + +void MtProgress_Init(CMtProgress *p, ICompressProgress *progress); +SRes MtProgress_Progress_ST(CMtProgress *p); +SRes MtProgress_ProgressAdd(CMtProgress *p, UInt64 inSize, UInt64 outSize); +SRes MtProgress_GetError(CMtProgress *p); +void MtProgress_SetError(CMtProgress *p, SRes res); + +struct _CMtDec; + +typedef struct +{ + struct _CMtDec *mtDec; + unsigned index; + void *inBuf; + + size_t inDataSize_Start; // size of input data in start block + UInt64 inDataSize; // total size of input data in all blocks + + CThread thread; + CAutoResetEvent canRead; + CAutoResetEvent canWrite; + void *allocaPtr; +} CMtDecThread; + +void MtDecThread_FreeInBufs(CMtDecThread *t); + + +typedef enum +{ + MTDEC_PARSE_CONTINUE, // continue this block with more input data + MTDEC_PARSE_OVERFLOW, // MT buffers overflow, need switch to single-thread + MTDEC_PARSE_NEW, // new block + MTDEC_PARSE_END // end of block threading. But we still can return to threading after Write(&needContinue) +} EMtDecParseState; + +typedef struct +{ + // in + int startCall; + const Byte *src; + size_t srcSize; + // in : (srcSize == 0) is allowed + // out : it's allowed to return less that actually was used ? + int srcFinished; + + // out + EMtDecParseState state; + Bool canCreateNewThread; + UInt64 outPos; // check it (size_t) +} CMtDecCallbackInfo; + + +typedef struct +{ + void (*Parse)(void *p, unsigned coderIndex, CMtDecCallbackInfo *ci); + + // PreCode() and Code(): + // (SRes_return_result != SZ_OK) means stop decoding, no need another blocks + SRes (*PreCode)(void *p, unsigned coderIndex); + SRes (*Code)(void *p, unsigned coderIndex, + const Byte *src, size_t srcSize, int srcFinished, + UInt64 *inCodePos, UInt64 *outCodePos, int *stop); + // stop - means stop another Code calls + + + /* Write() must be called, if Parse() was called + set (needWrite) if + { + && (was not interrupted by progress) + && (was not interrupted in previous block) + } + + out: + if (*needContinue), decoder still need to continue decoding with new iteration, + even after MTDEC_PARSE_END + if (*canRecode), we didn't flush current block data, so we still can decode current block later. + */ + SRes (*Write)(void *p, unsigned coderIndex, + Bool needWriteToStream, + const Byte *src, size_t srcSize, + // int srcFinished, + Bool *needContinue, + Bool *canRecode); +} IMtDecCallback; + + + +typedef struct _CMtDec +{ + /* input variables */ + + size_t inBufSize; /* size of input block */ + unsigned numThreadsMax; + // size_t inBlockMax; + unsigned numThreadsMax_2; + + ISeqInStream *inStream; + // const Byte *inData; + // size_t inDataSize; + + ICompressProgress *progress; + ISzAllocPtr alloc; + + IMtDecCallback *mtCallback; + void *mtCallbackObject; + + + /* internal variables */ + + size_t allocatedBufsSize; + + Bool exitThread; + WRes exitThreadWRes; + + UInt64 blockIndex; + Bool isAllocError; + Bool overflow; + SRes threadingErrorSRes; + + Bool needContinue; + + // CAutoResetEvent finishedEvent; + + SRes readRes; + SRes codeRes; + + Bool wasInterrupted; + + unsigned numStartedThreads_Limit; + unsigned numStartedThreads; + + Byte *crossBlock; + size_t crossStart; + size_t crossEnd; + UInt64 readProcessed; + Bool readWasFinished; + UInt64 inProcessed; + + unsigned filledThreadStart; + unsigned numFilledThreads; + + #ifndef _7ZIP_ST + Bool needInterrupt; + UInt64 interruptIndex; + CMtProgress mtProgress; + CMtDecThread threads[MTDEC__THREADS_MAX]; + #endif +} CMtDec; + + +void MtDec_Construct(CMtDec *p); +void MtDec_Destruct(CMtDec *p); + +/* +MtDec_Code() returns: + SZ_OK - in most cases + MY_SRes_HRESULT_FROM_WRes(WRes_error) - in case of unexpected error in threading function +*/ + +SRes MtDec_Code(CMtDec *p); +Byte *MtDec_GetCrossBuff(CMtDec *p); + +int MtDec_PrepareRead(CMtDec *p); +const Byte *MtDec_Read(CMtDec *p, size_t *inLim); + +#endif + +EXTERN_C_END + +#endif diff --git a/C/Util/7z/7zMain.c b/C/Util/7z/7zMain.c index d12ff79e..82aac89e 100644 --- a/C/Util/7z/7zMain.c +++ b/C/Util/7z/7zMain.c @@ -1,5 +1,5 @@ /* 7zMain.c - Test application for 7z Decoder -2017-08-26 : Igor Pavlov : Public domain */ +2018-04-19 : Igor Pavlov : Public domain */ #include "Precomp.h" @@ -606,6 +606,31 @@ int MY_CDECL main(int numargs, char *args[]) res = SZ_ERROR_FAIL; break; } + + #ifdef USE_WINDOWS_FILE + { + FILETIME mtime, ctime; + FILETIME *mtimePtr = NULL; + FILETIME *ctimePtr = NULL; + + if (SzBitWithVals_Check(&db.MTime, i)) + { + const CNtfsFileTime *t = &db.MTime.Vals[i]; + mtime.dwLowDateTime = (DWORD)(t->Low); + mtime.dwHighDateTime = (DWORD)(t->High); + mtimePtr = &mtime; + } + if (SzBitWithVals_Check(&db.CTime, i)) + { + const CNtfsFileTime *t = &db.CTime.Vals[i]; + ctime.dwLowDateTime = (DWORD)(t->Low); + ctime.dwHighDateTime = (DWORD)(t->High); + ctimePtr = &ctime; + } + if (mtimePtr || ctimePtr) + SetFileTime(outFile.handle, ctimePtr, NULL, mtimePtr); + } + #endif if (File_Close(&outFile)) { diff --git a/C/Util/7zipInstall/makefile b/C/Util/7zipInstall/makefile index c407479c..ac696403 100644 --- a/C/Util/7zipInstall/makefile +++ b/C/Util/7zipInstall/makefile @@ -1,4 +1,5 @@ PROG = 7zipInstall.exe +MY_FIXED = 1 LIBS = $(LIBS) version.lib diff --git a/C/Util/7zipUninstall/7zipUninstall.c b/C/Util/7zipUninstall/7zipUninstall.c index fe6f9b0a..381a5574 100644 --- a/C/Util/7zipUninstall/7zipUninstall.c +++ b/C/Util/7zipUninstall/7zipUninstall.c @@ -1,5 +1,5 @@ /* 7zipUninstall.c - 7-Zip Uninstaller -2017-04-04 : Igor Pavlov : Public domain */ +2018-03-01 : Igor Pavlov : Public domain */ #include "Precomp.h" @@ -546,7 +546,7 @@ static BOOL RemoveDir() #define k_Lang L"Lang" // NUM_LANG_TXT_FILES files are placed before en.ttt -#define NUM_LANG_TXT_FILES 87 +#define NUM_LANG_TXT_FILES 88 #ifdef _64BIT_INSTALLER #define NUM_EXTRA_FILES_64BIT 1 @@ -558,7 +558,7 @@ static BOOL RemoveDir() static const char * const k_Names = "af an ar ast az ba be bg bn br ca co cs cy da de el eo es et eu ext" - " fa fi fr fur fy ga gl gu he hi hr hu hy id io is it ja ka kaa kk ko ku ku-ckb ky" + " fa fi fr fur fy ga gl gu he hi hr hu hy id io is it ja ka kaa kab kk ko ku ku-ckb ky" " lij lt lv mk mn mng mng2 mr ms nb ne nl nn pa-in pl ps pt pt-br ro ru" " sa si sk sl sq sr-spc sr-spl sv ta th tr tt ug uk uz va vi yo zh-cn zh-tw" " en.ttt" diff --git a/C/Util/7zipUninstall/makefile b/C/Util/7zipUninstall/makefile index da44b850..823759a2 100644 --- a/C/Util/7zipUninstall/makefile +++ b/C/Util/7zipUninstall/makefile @@ -1,4 +1,5 @@ PROG = 7zipUninstall.exe +MY_FIXED = 1 !IFDEF _64BIT_INSTALLER CFLAGS = $(CFLAGS) -D_64BIT_INSTALLER diff --git a/C/Util/SfxSetup/makefile b/C/Util/SfxSetup/makefile index 6985944c..c9f59ccd 100644 --- a/C/Util/SfxSetup/makefile +++ b/C/Util/SfxSetup/makefile @@ -1,4 +1,5 @@ PROG = 7zS2.sfx +MY_FIXED = 1 C_OBJS = \ $O\7zAlloc.obj \ diff --git a/C/Util/SfxSetup/makefile_con b/C/Util/SfxSetup/makefile_con index cb2c1a43..6f604ed8 100644 --- a/C/Util/SfxSetup/makefile_con +++ b/C/Util/SfxSetup/makefile_con @@ -1,4 +1,5 @@ PROG = 7zS2con.sfx +MY_FIXED = 1 CFLAGS = $(CFLAGS) -D_CONSOLE C_OBJS = \ diff --git a/C/Xz.h b/C/Xz.h index ba455c13..7b88f516 100644 --- a/C/Xz.h +++ b/C/Xz.h @@ -1,5 +1,5 @@ /* Xz.h - Xz interface -2017-07-27 : Igor Pavlov : Public domain */ +2018-02-28 : Igor Pavlov : Public domain */ #ifndef __XZ_H #define __XZ_H @@ -139,6 +139,9 @@ SRes Xzs_ReadBackward(CXzs *p, ILookInStream *inStream, Int64 *startOffset, ICom UInt64 Xzs_GetNumBlocks(const CXzs *p); UInt64 Xzs_GetUnpackSize(const CXzs *p); + +// ECoderStatus values are identical to ELzmaStatus values of LZMA2 decoder + typedef enum { CODER_STATUS_NOT_SPECIFIED, /* use main error code instead */ @@ -147,22 +150,31 @@ typedef enum CODER_STATUS_NEEDS_MORE_INPUT /* you must provide more input bytes */ } ECoderStatus; + +// ECoderFinishMode values are identical to ELzmaFinishMode + typedef enum { CODER_FINISH_ANY, /* finish at any point */ CODER_FINISH_END /* block must be finished at the end */ } ECoderFinishMode; + typedef struct _IStateCoder { void *p; void (*Free)(void *p, ISzAllocPtr alloc); SRes (*SetProps)(void *p, const Byte *props, size_t propSize, ISzAllocPtr alloc); void (*Init)(void *p); - SRes (*Code)(void *p, Byte *dest, SizeT *destLen, const Byte *src, SizeT *srcLen, - int srcWasFinished, ECoderFinishMode finishMode, int *wasFinished); + SRes (*Code2)(void *p, Byte *dest, SizeT *destLen, const Byte *src, SizeT *srcLen, + int srcWasFinished, ECoderFinishMode finishMode, + // int *wasFinished, + ECoderStatus *status); + SizeT (*Filter)(void *p, Byte *data, SizeT size); } IStateCoder; + + #define MIXCODER_NUM_FILTERS_MAX 4 typedef struct @@ -170,20 +182,23 @@ typedef struct ISzAllocPtr alloc; Byte *buf; unsigned numCoders; + + Byte *outBuf; + size_t outBufSize; + size_t outWritten; // is equal to lzmaDecoder.dicPos (in outBuf mode) + Bool wasFinished; + SRes res; + ECoderStatus status; + // Bool SingleBufMode; + int finished[MIXCODER_NUM_FILTERS_MAX - 1]; size_t pos[MIXCODER_NUM_FILTERS_MAX - 1]; size_t size[MIXCODER_NUM_FILTERS_MAX - 1]; UInt64 ids[MIXCODER_NUM_FILTERS_MAX]; + SRes results[MIXCODER_NUM_FILTERS_MAX]; IStateCoder coders[MIXCODER_NUM_FILTERS_MAX]; } CMixCoder; -void MixCoder_Construct(CMixCoder *p, ISzAllocPtr alloc); -void MixCoder_Free(CMixCoder *p); -void MixCoder_Init(CMixCoder *p); -SRes MixCoder_SetFromMethod(CMixCoder *p, unsigned coderIndex, UInt64 methodId); -SRes MixCoder_Code(CMixCoder *p, Byte *dest, SizeT *destLen, - const Byte *src, SizeT *srcLen, int srcWasFinished, - ECoderFinishMode finishMode, ECoderStatus *status); typedef enum { @@ -197,6 +212,7 @@ typedef enum XZ_STATE_BLOCK_FOOTER } EXzState; + typedef struct { EXzState state; @@ -210,7 +226,7 @@ typedef struct UInt64 packSize; UInt64 unpackSize; - UInt64 numBlocks; + UInt64 numBlocks; // number of finished blocks in current stream UInt64 indexSize; UInt64 indexPos; UInt64 padSize; @@ -225,16 +241,64 @@ typedef struct CXzCheck check; CSha256 sha; + Bool parseMode; + Bool headerParsedOk; + Bool decodeToStreamSignature; unsigned decodeOnlyOneBlock; + Byte *outBuf; + size_t outBufSize; + size_t outDataWritten; // the size of data in (outBuf) that were fully unpacked + Byte shaDigest[SHA256_DIGEST_SIZE]; Byte buf[XZ_BLOCK_HEADER_SIZE_MAX]; } CXzUnpacker; +/* alloc : aligned for cache line allocation is better */ void XzUnpacker_Construct(CXzUnpacker *p, ISzAllocPtr alloc); void XzUnpacker_Init(CXzUnpacker *p); +void XzUnpacker_SetOutBuf(CXzUnpacker *p, Byte *outBuf, size_t outBufSize); void XzUnpacker_Free(CXzUnpacker *p); +/* + XzUnpacker + The sequence for decoding functions: + { + XzUnpacker_Construct() + [Decoding_Calls] + XzUnpacker_Free() + } + + [Decoding_Calls] + + There are 3 types of interfaces for [Decoding_Calls] calls: + + Interface-1 : Partial output buffers: + { + XzUnpacker_Init() + for() + XzUnpacker_Code(); + } + + Interface-2 : Direct output buffer: + Use it, if you know exact size of decoded data, and you need + whole xz unpacked data in one output buffer. + xz unpacker doesn't allocate additional buffer for lzma2 dictionary in that mode. + { + XzUnpacker_Init() + XzUnpacker_SetOutBufMode(); // to set output buffer and size + for() + XzUnpacker_Code(); // (dest = NULL) in XzUnpacker_Code() + } + + Interface-3 : Direct output buffer : One call full decoding + It unpacks whole input buffer to output buffer in one call. + It uses Interface-2 internally. + { + XzUnpacker_CodeFull() + } +*/ + /* finishMode: It has meaning only if the decoding reaches output limit (*destLen). @@ -264,15 +328,23 @@ Returns: SRes XzUnpacker_Code(CXzUnpacker *p, Byte *dest, SizeT *destLen, - const Byte *src, SizeT *srcLen, ECoderFinishMode finishMode, - ECoderStatus *status); + const Byte *src, SizeT *srcLen, int srcFinished, + ECoderFinishMode finishMode, ECoderStatus *status); + +SRes XzUnpacker_CodeFull(CXzUnpacker *p, Byte *dest, SizeT *destLen, + const Byte *src, SizeT *srcLen, + ECoderFinishMode finishMode, ECoderStatus *status); Bool XzUnpacker_IsStreamWasFinished(const CXzUnpacker *p); /* -Call XzUnpacker_GetExtraSize after XzUnpacker_Code function to detect real size of -xz stream in two cases: -XzUnpacker_Code() returns: +XzUnpacker_GetExtraSize() returns then number of uncofirmed bytes, + if it's in (XZ_STATE_STREAM_HEADER) state or in (XZ_STATE_STREAM_PADDING) state. +These bytes can be some bytes after xz archive, or +it can be start of new xz stream. + +Call XzUnpacker_GetExtraSize() after XzUnpacker_Code() function to detect real size of +xz stream in two cases, if XzUnpacker_Code() returns: res == SZ_OK && status == CODER_STATUS_NEEDS_MORE_INPUT res == SZ_ERROR_NO_ARCHIVE */ @@ -297,6 +369,92 @@ Bool XzUnpacker_IsBlockFinished(const CXzUnpacker *p); #define XzUnpacker_GetPackSizeForIndex(p) ((p)->packSize + (p)->blockHeaderSize + XzFlags_GetCheckSize((p)->streamFlags)) + + +/* ---------- Multi Threading Decoding ---------- */ + + +typedef struct +{ + size_t inBufSize_ST; + size_t outStep_ST; + Bool ignoreErrors; + + #ifndef _7ZIP_ST + unsigned numThreads; + size_t inBufSize_MT; + size_t memUseMax; + #endif +} CXzDecMtProps; + +void XzDecMtProps_Init(CXzDecMtProps *p); + + +typedef void * CXzDecMtHandle; + +/* + alloc : XzDecMt uses CAlignOffsetAlloc for addresses allocated by (alloc). + allocMid : for big allocations, aligned allocation is better +*/ + +CXzDecMtHandle XzDecMt_Create(ISzAllocPtr alloc, ISzAllocPtr allocMid); +void XzDecMt_Destroy(CXzDecMtHandle p); + + +typedef struct +{ + Byte UnpackSize_Defined; + Byte NumStreams_Defined; + Byte NumBlocks_Defined; + + Byte DataAfterEnd; + Byte DecodingTruncated; // Decoding was Truncated, we need only partial output data + + UInt64 InSize; // pack size processed + UInt64 OutSize; + + UInt64 NumStreams; + UInt64 NumBlocks; + + SRes DecodeRes; + SRes ReadRes; + SRes ProgressRes; + SRes CombinedRes; + SRes CombinedRes_Type; + +} CXzStatInfo; + +void XzStatInfo_Clear(CXzStatInfo *p); + +/* +XzDecMt_Decode() +SRes: + SZ_OK - OK + SZ_ERROR_MEM - Memory allocation error + SZ_ERROR_NO_ARCHIVE - is not xz archive + SZ_ERROR_ARCHIVE - Headers error + SZ_ERROR_DATA - Data Error + SZ_ERROR_CRC - CRC Error + SZ_ERROR_INPUT_EOF - it needs more input data + SZ_ERROR_WRITE - ISeqOutStream error + (SZ_ERROR_READ) - ISeqInStream errors + (SZ_ERROR_PROGRESS) - ICompressProgress errors + // SZ_ERROR_THREAD - error in multi-threading functions + MY_SRes_HRESULT_FROM_WRes(WRes_error) - error in multi-threading function +*/ + +SRes XzDecMt_Decode(CXzDecMtHandle p, + const CXzDecMtProps *props, + const UInt64 *outDataSize, // NULL means undefined + int finishMode, // 0 - partial unpacking is allowed, 1 - xz stream(s) must be finished + ISeqOutStream *outStream, + // Byte *outBuf, size_t *outBufSize, + ISeqInStream *inStream, + // const Byte *inData, size_t inDataSize, + CXzStatInfo *stat, + int *isMT, // 0 means that ST (Single-Thread) version was used + ICompressProgress *progress); + EXTERN_C_END #endif diff --git a/C/XzDec.c b/C/XzDec.c index 7747ba44..ebf19833 100644 --- a/C/XzDec.c +++ b/C/XzDec.c @@ -1,14 +1,33 @@ /* XzDec.c -- Xz Decode -2017-07-27 : Igor Pavlov : Public domain */ +2018-04-24 : Igor Pavlov : Public domain */ #include "Precomp.h" +// #include + +// #define XZ_DUMP + /* #define XZ_DUMP */ #ifdef XZ_DUMP #include #endif +// #define SHOW_DEBUG_INFO + +#ifdef SHOW_DEBUG_INFO +#include +#endif + +#ifdef SHOW_DEBUG_INFO +#define PRF(x) x +#else +#define PRF(x) +#endif + +#define PRF_STR(s) PRF(printf("\n" s "\n")) +#define PRF_STR_INT(s, d) PRF(printf("\n" s " %d\n", (unsigned)d)) + #include #include @@ -19,9 +38,11 @@ #include "Delta.h" #include "Lzma2Dec.h" +// #define USE_SUBBLOCK + #ifdef USE_SUBBLOCK #include "Bcj3Dec.c" -#include "SbDec.c" +#include "SbDec.h" #endif #include "Xz.h" @@ -56,8 +77,9 @@ typedef struct size_t bufConv; size_t bufTotal; - UInt32 methodId; int encodeMode; + + UInt32 methodId; UInt32 delta; UInt32 ip; UInt32 x86State; @@ -121,69 +143,84 @@ static void BraState_Init(void *pp) Delta_Init(p->deltaState); } -#define CASE_BRA_CONV(isa) case XZ_ID_ ## isa: p->bufConv = isa ## _Convert(p->buf, p->bufTotal, p->ip, p->encodeMode); break; -static SRes BraState_Code(void *pp, Byte *dest, SizeT *destLen, const Byte *src, SizeT *srcLen, - int srcWasFinished, ECoderFinishMode finishMode, int *wasFinished) +#define CASE_BRA_CONV(isa) case XZ_ID_ ## isa: size = isa ## _Convert(data, size, p->ip, p->encodeMode); break; + +static SizeT BraState_Filter(void *pp, Byte *data, SizeT size) { CBraState *p = ((CBraState *)pp); - SizeT destLenOrig = *destLen; - SizeT srcLenOrig = *srcLen; + switch (p->methodId) + { + case XZ_ID_Delta: + if (p->encodeMode) + Delta_Encode(p->deltaState, p->delta, data, size); + else + Delta_Decode(p->deltaState, p->delta, data, size); + break; + case XZ_ID_X86: + size = x86_Convert(data, size, p->ip, &p->x86State, p->encodeMode); + break; + CASE_BRA_CONV(PPC) + CASE_BRA_CONV(IA64) + CASE_BRA_CONV(ARM) + CASE_BRA_CONV(ARMT) + CASE_BRA_CONV(SPARC) + } + p->ip += (UInt32)size; + return size; +} + + +static SRes BraState_Code2(void *pp, + Byte *dest, SizeT *destLen, + const Byte *src, SizeT *srcLen, int srcWasFinished, + ECoderFinishMode finishMode, + // int *wasFinished + ECoderStatus *status) +{ + CBraState *p = ((CBraState *)pp); + SizeT destRem = *destLen; + SizeT srcRem = *srcLen; UNUSED_VAR(finishMode); + *destLen = 0; *srcLen = 0; - *wasFinished = 0; - while (destLenOrig > 0) + // *wasFinished = False; + *status = CODER_STATUS_NOT_FINISHED; + + while (destRem > 0) { if (p->bufPos != p->bufConv) { - size_t curSize = p->bufConv - p->bufPos; - if (curSize > destLenOrig) - curSize = destLenOrig; - memcpy(dest, p->buf + p->bufPos, curSize); - p->bufPos += curSize; - *destLen += curSize; - dest += curSize; - destLenOrig -= curSize; + size_t size = p->bufConv - p->bufPos; + if (size > destRem) + size = destRem; + memcpy(dest, p->buf + p->bufPos, size); + p->bufPos += size; + *destLen += size; + dest += size; + destRem -= size; continue; } + p->bufTotal -= p->bufPos; memmove(p->buf, p->buf + p->bufPos, p->bufTotal); p->bufPos = 0; p->bufConv = 0; { - size_t curSize = BRA_BUF_SIZE - p->bufTotal; - if (curSize > srcLenOrig) - curSize = srcLenOrig; - memcpy(p->buf + p->bufTotal, src, curSize); - *srcLen += curSize; - src += curSize; - srcLenOrig -= curSize; - p->bufTotal += curSize; + size_t size = BRA_BUF_SIZE - p->bufTotal; + if (size > srcRem) + size = srcRem; + memcpy(p->buf + p->bufTotal, src, size); + *srcLen += size; + src += size; + srcRem -= size; + p->bufTotal += size; } if (p->bufTotal == 0) break; - switch (p->methodId) - { - case XZ_ID_Delta: - if (p->encodeMode) - Delta_Encode(p->deltaState, p->delta, p->buf, p->bufTotal); - else - Delta_Decode(p->deltaState, p->delta, p->buf, p->bufTotal); - p->bufConv = p->bufTotal; - break; - case XZ_ID_X86: - p->bufConv = x86_Convert(p->buf, p->bufTotal, p->ip, &p->x86State, p->encodeMode); - break; - CASE_BRA_CONV(PPC) - CASE_BRA_CONV(IA64) - CASE_BRA_CONV(ARM) - CASE_BRA_CONV(ARMT) - CASE_BRA_CONV(SPARC) - default: - return SZ_ERROR_UNSUPPORTED; - } - p->ip += (UInt32)p->bufConv; + + p->bufConv = BraState_Filter(pp, p->buf, p->bufTotal); if (p->bufConv == 0) { @@ -192,36 +229,42 @@ static SRes BraState_Code(void *pp, Byte *dest, SizeT *destLen, const Byte *src, p->bufConv = p->bufTotal; } } - if (p->bufTotal == p->bufPos && srcLenOrig == 0 && srcWasFinished) - *wasFinished = 1; + + if (p->bufTotal == p->bufPos && srcRem == 0 && srcWasFinished) + { + *status = CODER_STATUS_FINISHED_WITH_MARK; + // *wasFinished = 1; + } + return SZ_OK; } + SRes BraState_SetFromMethod(IStateCoder *p, UInt64 id, int encodeMode, ISzAllocPtr alloc) { CBraState *decoder; - if (id != XZ_ID_Delta && - id != XZ_ID_X86 && - id != XZ_ID_PPC && - id != XZ_ID_IA64 && - id != XZ_ID_ARM && - id != XZ_ID_ARMT && - id != XZ_ID_SPARC) + if (id < XZ_ID_Delta || id > XZ_ID_SPARC) return SZ_ERROR_UNSUPPORTED; - p->p = 0; - decoder = (CBraState *)ISzAlloc_Alloc(alloc, sizeof(CBraState)); + decoder = p->p; if (!decoder) - return SZ_ERROR_MEM; + { + decoder = (CBraState *)ISzAlloc_Alloc(alloc, sizeof(CBraState)); + if (!decoder) + return SZ_ERROR_MEM; + p->p = decoder; + p->Free = BraState_Free; + p->SetProps = BraState_SetProps; + p->Init = BraState_Init; + p->Code2 = BraState_Code2; + p->Filter = BraState_Filter; + } decoder->methodId = (UInt32)id; decoder->encodeMode = encodeMode; - p->p = decoder; - p->Free = BraState_Free; - p->SetProps = BraState_SetProps; - p->Init = BraState_Init; - p->Code = BraState_Code; return SZ_OK; } + + /* ---------- SbState ---------- */ #ifdef USE_SUBBLOCK @@ -246,8 +289,10 @@ static void SbState_Init(void *pp) SbDec_Init((CSbDec *)pp); } -static SRes SbState_Code(void *pp, Byte *dest, SizeT *destLen, const Byte *src, SizeT *srcLen, - int srcWasFinished, ECoderFinishMode finishMode, int *wasFinished) +static SRes SbState_Code2(void *pp, Byte *dest, SizeT *destLen, const Byte *src, SizeT *srcLen, + int srcWasFinished, ECoderFinishMode finishMode, + // int *wasFinished + ECoderStatus *status) { CSbDec *p = (CSbDec *)pp; SRes res; @@ -260,33 +305,53 @@ static SRes SbState_Code(void *pp, Byte *dest, SizeT *destLen, const Byte *src, res = SbDec_Decode((CSbDec *)pp); *destLen -= p->destLen; *srcLen -= p->srcLen; - *wasFinished = (*destLen == 0 && *srcLen == 0); /* change it */ + // *wasFinished = (*destLen == 0 && *srcLen == 0); /* change it */ + *status = (*destLen == 0 && *srcLen == 0) ? + CODER_STATUS_FINISHED_WITH_MARK : + CODER_STATUS_NOT_FINISHED; return res; } -SRes SbState_SetFromMethod(IStateCoder *p, ISzAllocPtr alloc) +static SRes SbState_SetFromMethod(IStateCoder *p, ISzAllocPtr alloc) { - CSbDec *decoder; - p->p = 0; - decoder = ISzAlloc_Alloc(alloc, sizeof(CSbDec)); + CSbDec *decoder = (CSbDec *)p->p; if (!decoder) - return SZ_ERROR_MEM; - p->p = decoder; - p->Free = SbState_Free; - p->SetProps = SbState_SetProps; - p->Init = SbState_Init; - p->Code = SbState_Code; + { + decoder = (CSbDec *)ISzAlloc_Alloc(alloc, sizeof(CSbDec)); + if (!decoder) + return SZ_ERROR_MEM; + p->p = decoder; + p->Free = SbState_Free; + p->SetProps = SbState_SetProps; + p->Init = SbState_Init; + p->Code2 = SbState_Code2; + p->Filter = NULL; + } SbDec_Construct(decoder); SbDec_SetAlloc(decoder, alloc); return SZ_OK; } + #endif -/* ---------- Lzma2State ---------- */ + + +/* ---------- Lzma2 ---------- */ + +typedef struct +{ + CLzma2Dec decoder; + Bool outBufMode; +} CLzma2Dec_Spec; + static void Lzma2State_Free(void *pp, ISzAllocPtr alloc) { - Lzma2Dec_Free((CLzma2Dec *)pp, alloc); + CLzma2Dec_Spec *p = (CLzma2Dec_Spec *)pp; + if (p->outBufMode) + Lzma2Dec_FreeProbs(&p->decoder, alloc); + else + Lzma2Dec_Free(&p->decoder, alloc); ISzAlloc_Free(alloc, pp); } @@ -294,60 +359,123 @@ static SRes Lzma2State_SetProps(void *pp, const Byte *props, size_t propSize, IS { if (propSize != 1) return SZ_ERROR_UNSUPPORTED; - return Lzma2Dec_Allocate((CLzma2Dec *)pp, props[0], alloc); + { + CLzma2Dec_Spec *p = (CLzma2Dec_Spec *)pp; + if (p->outBufMode) + return Lzma2Dec_AllocateProbs(&p->decoder, props[0], alloc); + else + return Lzma2Dec_Allocate(&p->decoder, props[0], alloc); + } } static void Lzma2State_Init(void *pp) { - Lzma2Dec_Init((CLzma2Dec *)pp); + Lzma2Dec_Init(&((CLzma2Dec_Spec *)pp)->decoder); } -static SRes Lzma2State_Code(void *pp, Byte *dest, SizeT *destLen, const Byte *src, SizeT *srcLen, - int srcWasFinished, ECoderFinishMode finishMode, int *wasFinished) + +/* + if (outBufMode), then (dest) is not used. Use NULL. + Data is unpacked to (spec->decoder.decoder.dic) output buffer. +*/ + +static SRes Lzma2State_Code2(void *pp, Byte *dest, SizeT *destLen, const Byte *src, SizeT *srcLen, + int srcWasFinished, ECoderFinishMode finishMode, + // int *wasFinished, + ECoderStatus *status) { - ELzmaStatus status; + CLzma2Dec_Spec *spec = (CLzma2Dec_Spec *)pp; + ELzmaStatus status2; /* ELzmaFinishMode fm = (finishMode == LZMA_FINISH_ANY) ? LZMA_FINISH_ANY : LZMA_FINISH_END; */ - SRes res = Lzma2Dec_DecodeToBuf((CLzma2Dec *)pp, dest, destLen, src, srcLen, (ELzmaFinishMode)finishMode, &status); + SRes res; UNUSED_VAR(srcWasFinished); - *wasFinished = (status == LZMA_STATUS_FINISHED_WITH_MARK); + if (spec->outBufMode) + { + SizeT dicPos = spec->decoder.decoder.dicPos; + SizeT dicLimit = dicPos + *destLen; + res = Lzma2Dec_DecodeToDic(&spec->decoder, dicLimit, src, srcLen, (ELzmaFinishMode)finishMode, &status2); + *destLen = spec->decoder.decoder.dicPos - dicPos; + } + else + res = Lzma2Dec_DecodeToBuf(&spec->decoder, dest, destLen, src, srcLen, (ELzmaFinishMode)finishMode, &status2); + // *wasFinished = (status2 == LZMA_STATUS_FINISHED_WITH_MARK); + // ECoderStatus values are identical to ELzmaStatus values of LZMA2 decoder + *status = status2; return res; } -static SRes Lzma2State_SetFromMethod(IStateCoder *p, ISzAllocPtr alloc) + +static SRes Lzma2State_SetFromMethod(IStateCoder *p, Byte *outBuf, size_t outBufSize, ISzAllocPtr alloc) { - CLzma2Dec *decoder = (CLzma2Dec *)ISzAlloc_Alloc(alloc, sizeof(CLzma2Dec)); - p->p = decoder; - if (!decoder) - return SZ_ERROR_MEM; - p->Free = Lzma2State_Free; - p->SetProps = Lzma2State_SetProps; - p->Init = Lzma2State_Init; - p->Code = Lzma2State_Code; - Lzma2Dec_Construct(decoder); + CLzma2Dec_Spec *spec = (CLzma2Dec_Spec *)p->p; + if (!spec) + { + spec = (CLzma2Dec_Spec *)ISzAlloc_Alloc(alloc, sizeof(CLzma2Dec_Spec)); + if (!spec) + return SZ_ERROR_MEM; + p->p = spec; + p->Free = Lzma2State_Free; + p->SetProps = Lzma2State_SetProps; + p->Init = Lzma2State_Init; + p->Code2 = Lzma2State_Code2; + p->Filter = NULL; + Lzma2Dec_Construct(&spec->decoder); + } + spec->outBufMode = False; + if (outBuf) + { + spec->outBufMode = True; + spec->decoder.decoder.dic = outBuf; + spec->decoder.decoder.dicBufSize = outBufSize; + } return SZ_OK; } -void MixCoder_Construct(CMixCoder *p, ISzAllocPtr alloc) +static SRes Lzma2State_ResetOutBuf(IStateCoder *p, Byte *outBuf, size_t outBufSize) +{ + CLzma2Dec_Spec *spec = (CLzma2Dec_Spec *)p->p; + if ((spec->outBufMode && !outBuf) || (!spec->outBufMode && outBuf)) + return SZ_ERROR_FAIL; + if (outBuf) + { + spec->decoder.decoder.dic = outBuf; + spec->decoder.decoder.dicBufSize = outBufSize; + } + return SZ_OK; +} + + + +static void MixCoder_Construct(CMixCoder *p, ISzAllocPtr alloc) { unsigned i; p->alloc = alloc; p->buf = NULL; p->numCoders = 0; + + p->outBufSize = 0; + p->outBuf = NULL; + // p->SingleBufMode = False; + for (i = 0; i < MIXCODER_NUM_FILTERS_MAX; i++) p->coders[i].p = NULL; } -void MixCoder_Free(CMixCoder *p) + +static void MixCoder_Free(CMixCoder *p) { unsigned i; - for (i = 0; i < p->numCoders; i++) + p->numCoders = 0; + for (i = 0; i < MIXCODER_NUM_FILTERS_MAX; i++) { IStateCoder *sc = &p->coders[i]; - if (p->alloc && sc->p) + if (sc->p) + { sc->Free(sc->p, p->alloc); + sc->p = NULL; + } } - p->numCoders = 0; if (p->buf) { ISzAlloc_Free(p->alloc, p->buf); @@ -355,7 +483,7 @@ void MixCoder_Free(CMixCoder *p) } } -void MixCoder_Init(CMixCoder *p) +static void MixCoder_Init(CMixCoder *p) { unsigned i; for (i = 0; i < MIXCODER_NUM_FILTERS_MAX - 1; i++) @@ -368,16 +496,22 @@ void MixCoder_Init(CMixCoder *p) { IStateCoder *coder = &p->coders[i]; coder->Init(coder->p); + p->results[i] = SZ_OK; } + p->outWritten = 0; + p->wasFinished = False; + p->res = SZ_OK; + p->status = CODER_STATUS_NOT_SPECIFIED; } -SRes MixCoder_SetFromMethod(CMixCoder *p, unsigned coderIndex, UInt64 methodId) + +static SRes MixCoder_SetFromMethod(CMixCoder *p, unsigned coderIndex, UInt64 methodId, Byte *outBuf, size_t outBufSize) { IStateCoder *sc = &p->coders[coderIndex]; p->ids[coderIndex] = methodId; switch (methodId) { - case XZ_ID_LZMA2: return Lzma2State_SetFromMethod(sc, p->alloc); + case XZ_ID_LZMA2: return Lzma2State_SetFromMethod(sc, outBuf, outBufSize, p->alloc); #ifdef USE_SUBBLOCK case XZ_ID_Subblock: return SbState_SetFromMethod(sc, p->alloc); #endif @@ -387,32 +521,128 @@ SRes MixCoder_SetFromMethod(CMixCoder *p, unsigned coderIndex, UInt64 methodId) return BraState_SetFromMethod(sc, methodId, 0, p->alloc); } -SRes MixCoder_Code(CMixCoder *p, Byte *dest, SizeT *destLen, + +static SRes MixCoder_ResetFromMethod(CMixCoder *p, unsigned coderIndex, UInt64 methodId, Byte *outBuf, size_t outBufSize) +{ + IStateCoder *sc = &p->coders[coderIndex]; + switch (methodId) + { + case XZ_ID_LZMA2: return Lzma2State_ResetOutBuf(sc, outBuf, outBufSize); + } + return SZ_ERROR_UNSUPPORTED; +} + + + +/* + if (destFinish) - then unpack data block is finished at (*destLen) position, + and we can return data that were not processed by filter + +output (status) can be : + CODER_STATUS_NOT_FINISHED + CODER_STATUS_FINISHED_WITH_MARK + CODER_STATUS_NEEDS_MORE_INPUT - not implemented still +*/ + +static SRes MixCoder_Code(CMixCoder *p, + Byte *dest, SizeT *destLen, int destFinish, const Byte *src, SizeT *srcLen, int srcWasFinished, - ECoderFinishMode finishMode, ECoderStatus *status) + ECoderFinishMode finishMode) { SizeT destLenOrig = *destLen; SizeT srcLenOrig = *srcLen; *destLen = 0; *srcLen = 0; - *status = CODER_STATUS_NOT_FINISHED; - if (!p->buf) + if (p->wasFinished) + return p->res; + + p->status = CODER_STATUS_NOT_FINISHED; + + // if (p->SingleBufMode) + if (p->outBuf) { - p->buf = (Byte *)ISzAlloc_Alloc(p->alloc, CODER_BUF_SIZE * (MIXCODER_NUM_FILTERS_MAX - 1)); - if (!p->buf) - return SZ_ERROR_MEM; + SRes res; + SizeT destLen2, srcLen2; + int wasFinished; + + PRF_STR("------- MixCoder Single ----------"); + + srcLen2 = srcLenOrig; + destLen2 = destLenOrig; + + { + IStateCoder *coder = &p->coders[0]; + res = coder->Code2(coder->p, NULL, &destLen2, src, &srcLen2, srcWasFinished, finishMode, + // &wasFinished, + &p->status); + wasFinished = (p->status == CODER_STATUS_FINISHED_WITH_MARK); + } + + p->res = res; + + /* + if (wasFinished) + p->status = CODER_STATUS_FINISHED_WITH_MARK; + else + { + if (res == SZ_OK) + if (destLen2 != destLenOrig) + p->status = CODER_STATUS_NEEDS_MORE_INPUT; + } + */ + + + *srcLen = srcLen2; + src += srcLen2; + p->outWritten += destLen2; + + if (res != SZ_OK || srcWasFinished || wasFinished) + p->wasFinished = True; + + if (p->numCoders == 1) + *destLen = destLen2; + else if (p->wasFinished) + { + unsigned i; + size_t processed = p->outWritten; + + for (i = 1; i < p->numCoders; i++) + { + IStateCoder *coder = &p->coders[i]; + processed = coder->Filter(coder->p, p->outBuf, processed); + if (wasFinished || (destFinish && p->outWritten == destLenOrig)) + processed = p->outWritten; + PRF_STR_INT("filter", i); + } + *destLen = processed; + } + return res; } + PRF_STR("standard mix"); + if (p->numCoders != 1) + { + if (!p->buf) + { + p->buf = (Byte *)ISzAlloc_Alloc(p->alloc, CODER_BUF_SIZE * (MIXCODER_NUM_FILTERS_MAX - 1)); + if (!p->buf) + return SZ_ERROR_MEM; + } + finishMode = CODER_FINISH_ANY; + } for (;;) { Bool processed = False; Bool allFinished = True; + SRes resMain = SZ_OK; unsigned i; + + p->status = CODER_STATUS_NOT_FINISHED; /* if (p->numCoders == 1 && *destLen == destLenOrig && finishMode == LZMA_FINISH_ANY) break; @@ -422,78 +652,101 @@ SRes MixCoder_Code(CMixCoder *p, Byte *dest, SizeT *destLen, { SRes res; IStateCoder *coder = &p->coders[i]; - Byte *destCur; - SizeT destLenCur, srcLenCur; - const Byte *srcCur; - int srcFinishedCur; + Byte *dest2; + SizeT destLen2, srcLen2; // destLen2_Orig; + const Byte *src2; + int srcFinished2; int encodingWasFinished; + ECoderStatus status2; if (i == 0) { - srcCur = src; - srcLenCur = srcLenOrig - *srcLen; - srcFinishedCur = srcWasFinished; + src2 = src; + srcLen2 = srcLenOrig - *srcLen; + srcFinished2 = srcWasFinished; } else { size_t k = i - 1; - srcCur = p->buf + (CODER_BUF_SIZE * k) + p->pos[k]; - srcLenCur = p->size[k] - p->pos[k]; - srcFinishedCur = p->finished[k]; + src2 = p->buf + (CODER_BUF_SIZE * k) + p->pos[k]; + srcLen2 = p->size[k] - p->pos[k]; + srcFinished2 = p->finished[k]; } if (i == p->numCoders - 1) { - destCur = dest; - destLenCur = destLenOrig - *destLen; + dest2 = dest; + destLen2 = destLenOrig - *destLen; } else { if (p->pos[i] != p->size[i]) continue; - destCur = p->buf + (CODER_BUF_SIZE * i); - destLenCur = CODER_BUF_SIZE; + dest2 = p->buf + (CODER_BUF_SIZE * i); + destLen2 = CODER_BUF_SIZE; } - res = coder->Code(coder->p, destCur, &destLenCur, srcCur, &srcLenCur, srcFinishedCur, finishMode, &encodingWasFinished); + // destLen2_Orig = destLen2; + + if (p->results[i] != SZ_OK) + { + if (resMain == SZ_OK) + resMain = p->results[i]; + continue; + } + res = coder->Code2(coder->p, + dest2, &destLen2, + src2, &srcLen2, srcFinished2, + finishMode, + // &encodingWasFinished, + &status2); + + if (res != SZ_OK) + { + p->results[i] = res; + if (resMain == SZ_OK) + resMain = res; + } + + encodingWasFinished = (status2 == CODER_STATUS_FINISHED_WITH_MARK); + if (!encodingWasFinished) + { allFinished = False; + if (p->numCoders == 1 && res == SZ_OK) + p->status = status2; + } if (i == 0) { - *srcLen += srcLenCur; - src += srcLenCur; + *srcLen += srcLen2; + src += srcLen2; } else - { - p->pos[(size_t)i - 1] += srcLenCur; - } + p->pos[(size_t)i - 1] += srcLen2; if (i == p->numCoders - 1) { - *destLen += destLenCur; - dest += destLenCur; + *destLen += destLen2; + dest += destLen2; } else { - p->size[i] = destLenCur; + p->size[i] = destLen2; p->pos[i] = 0; p->finished[i] = encodingWasFinished; } - if (res != SZ_OK) - return res; - - if (destLenCur != 0 || srcLenCur != 0) + if (destLen2 != 0 || srcLen2 != 0) processed = True; } if (!processed) { if (allFinished) - *status = CODER_STATUS_FINISHED_WITH_MARK; - return SZ_OK; + p->status = CODER_STATUS_FINISHED_WITH_MARK; + return resMain; } } } @@ -522,6 +775,33 @@ static Bool Xz_CheckFooter(CXzStreamFlags flags, UInt64 indexSize, const Byte *b if (s == 0) return SZ_ERROR_ARCHIVE; pos += s; } +static Bool XzBlock_AreSupportedFilters(const CXzBlock *p) +{ + unsigned numFilters = XzBlock_GetNumFilters(p) - 1; + unsigned i; + { + const CXzFilter *f = &p->filters[numFilters]; + if (f->id != XZ_ID_LZMA2 || f->propsSize != 1 || f->props[0] > 40) + return False; + } + + for (i = 0; i < numFilters; i++) + { + const CXzFilter *f = &p->filters[i]; + if (f->id == XZ_ID_Delta) + { + if (f->propsSize != 1) + return False; + } + else if (f->id < XZ_ID_Delta + || f->id > XZ_ID_SPARC + || (f->propsSize != 0 && f->propsSize != 4)) + return False; + } + return True; +} + + SRes XzBlock_Parse(CXzBlock *p, const Byte *header) { unsigned pos; @@ -580,31 +860,47 @@ SRes XzBlock_Parse(CXzBlock *p, const Byte *header) return SZ_OK; } -SRes XzDec_Init(CMixCoder *p, const CXzBlock *block) + + + +static SRes XzDecMix_Init(CMixCoder *p, const CXzBlock *block, Byte *outBuf, size_t outBufSize) { unsigned i; Bool needReInit = True; unsigned numFilters = XzBlock_GetNumFilters(block); - - if (numFilters == p->numCoders) + + if (numFilters == p->numCoders && ((p->outBuf && outBuf) || (!p->outBuf && !outBuf))) { + needReInit = False; for (i = 0; i < numFilters; i++) if (p->ids[i] != block->filters[numFilters - 1 - i].id) + { + needReInit = True; break; - needReInit = (i != numFilters); + } } + + // p->SingleBufMode = (outBuf != NULL); + p->outBuf = outBuf; + p->outBufSize = outBufSize; + + // p->SingleBufMode = False; + // outBuf = NULL; if (needReInit) { MixCoder_Free(p); - p->numCoders = numFilters; for (i = 0; i < numFilters; i++) { - const CXzFilter *f = &block->filters[numFilters - 1 - i]; - RINOK(MixCoder_SetFromMethod(p, i, f->id)); + RINOK(MixCoder_SetFromMethod(p, i, block->filters[numFilters - 1 - i].id, outBuf, outBufSize)); } + p->numCoders = numFilters; } - + else + { + RINOK(MixCoder_ResetFromMethod(p, 0, block->filters[numFilters - 1].id, outBuf, outBufSize)); + } + for (i = 0; i < numFilters; i++) { const CXzFilter *f = &block->filters[numFilters - 1 - i]; @@ -616,6 +912,8 @@ SRes XzDec_Init(CMixCoder *p, const CXzBlock *block) return SZ_OK; } + + void XzUnpacker_Init(CXzUnpacker *p) { p->state = XZ_STATE_STREAM_HEADER; @@ -625,14 +923,32 @@ void XzUnpacker_Init(CXzUnpacker *p) p->numTotalBlocks = 0; p->padSize = 0; p->decodeOnlyOneBlock = 0; + + p->parseMode = False; + p->decodeToStreamSignature = False; + + // p->outBuf = NULL; + // p->outBufSize = 0; + p->outDataWritten = 0; } + +void XzUnpacker_SetOutBuf(CXzUnpacker *p, Byte *outBuf, size_t outBufSize) +{ + p->outBuf = outBuf; + p->outBufSize = outBufSize; +} + + void XzUnpacker_Construct(CXzUnpacker *p, ISzAllocPtr alloc) { MixCoder_Construct(&p->decoder, alloc); + p->outBuf = NULL; + p->outBufSize = 0; XzUnpacker_Init(p); } + void XzUnpacker_Free(CXzUnpacker *p) { MixCoder_Free(&p->decoder); @@ -650,14 +966,28 @@ void XzUnpacker_PrepareToRandomBlockDecoding(CXzUnpacker *p) } +static void XzUnpacker_UpdateIndex(CXzUnpacker *p, UInt64 packSize, UInt64 unpackSize) +{ + Byte temp[32]; + unsigned num = Xz_WriteVarInt(temp, packSize); + num += Xz_WriteVarInt(temp + num, unpackSize); + Sha256_Update(&p->sha, temp, num); + p->indexSize += num; + p->numBlocks++; +} + + + SRes XzUnpacker_Code(CXzUnpacker *p, Byte *dest, SizeT *destLen, - const Byte *src, SizeT *srcLen, ECoderFinishMode finishMode, ECoderStatus *status) + const Byte *src, SizeT *srcLen, int srcFinished, + ECoderFinishMode finishMode, ECoderStatus *status) { SizeT destLenOrig = *destLen; SizeT srcLenOrig = *srcLen; *destLen = 0; *srcLen = 0; *status = CODER_STATUS_NOT_SPECIFIED; + for (;;) { SizeT srcRem; @@ -669,12 +999,17 @@ SRes XzUnpacker_Code(CXzUnpacker *p, Byte *dest, SizeT *destLen, SRes res; ECoderFinishMode finishMode2 = finishMode; + Bool srcFinished2 = srcFinished; + Bool destFinish = False; if (p->block.packSize != (UInt64)(Int64)-1) { UInt64 rem = p->block.packSize - p->packSize; - if (srcLen2 > rem) + if (srcLen2 >= rem) + { + srcFinished2 = True; srcLen2 = (SizeT)rem; + } if (rem == 0 && p->block.unpackSize == p->unpackSize) return SZ_ERROR_DATA; } @@ -684,6 +1019,7 @@ SRes XzUnpacker_Code(CXzUnpacker *p, Byte *dest, SizeT *destLen, UInt64 rem = p->block.unpackSize - p->unpackSize; if (destLen2 >= rem) { + destFinish = True; finishMode2 = CODER_FINISH_END; destLen2 = (SizeT)rem; } @@ -697,15 +1033,23 @@ SRes XzUnpacker_Code(CXzUnpacker *p, Byte *dest, SizeT *destLen, } */ - res = MixCoder_Code(&p->decoder, dest, &destLen2, src, &srcLen2, False, finishMode2, status); - XzCheck_Update(&p->check, dest, destLen2); + { + res = MixCoder_Code(&p->decoder, + (p->outBuf ? NULL : dest), &destLen2, destFinish, + src, &srcLen2, srcFinished2, + finishMode2); + + *status = p->decoder.status; + XzCheck_Update(&p->check, (p->outBuf ? p->outBuf + p->outDataWritten : dest), destLen2); + if (!p->outBuf) + dest += destLen2; + p->outDataWritten += destLen2; + } (*srcLen) += srcLen2; src += srcLen2; p->packSize += srcLen2; - (*destLen) += destLen2; - dest += destLen2; p->unpackSize += destLen2; RINOK(res); @@ -715,34 +1059,34 @@ SRes XzUnpacker_Code(CXzUnpacker *p, Byte *dest, SizeT *destLen, if (p->block.packSize == p->packSize && *status == CODER_STATUS_NEEDS_MORE_INPUT) { + PRF_STR("CODER_STATUS_NEEDS_MORE_INPUT"); *status = CODER_STATUS_NOT_SPECIFIED; return SZ_ERROR_DATA; } - // if (srcLen2 == 0 && destLen2 == 0) return SZ_OK; } { - Byte temp[32]; - unsigned num = Xz_WriteVarInt(temp, XzUnpacker_GetPackSizeForIndex(p)); - num += Xz_WriteVarInt(temp + num, p->unpackSize); - Sha256_Update(&p->sha, temp, num); - p->indexSize += num; - p->numBlocks++; + XzUnpacker_UpdateIndex(p, XzUnpacker_GetPackSizeForIndex(p), p->unpackSize); p->state = XZ_STATE_BLOCK_FOOTER; p->pos = 0; p->alignPos = 0; - - if (p->block.unpackSize != (UInt64)(Int64)-1) - if (p->block.unpackSize != p->unpackSize) - return SZ_ERROR_DATA; + *status = CODER_STATUS_NOT_SPECIFIED; + + if ((p->block.packSize != (UInt64)(Int64)-1 && p->block.packSize != p->packSize) + || (p->block.unpackSize != (UInt64)(Int64)-1 && p->block.unpackSize != p->unpackSize)) + { + PRF_STR("ERROR: block.size mismatch"); + return SZ_ERROR_DATA; + } } // continue; } srcRem = srcLenOrig - *srcLen; - if (srcRem == 0) + // XZ_STATE_BLOCK_FOOTER can transit to XZ_STATE_BLOCK_HEADER without input bytes + if (srcRem == 0 && p->state != XZ_STATE_BLOCK_FOOTER) { *status = CODER_STATUS_NEEDS_MORE_INPUT; return SZ_OK; @@ -756,6 +1100,8 @@ SRes XzUnpacker_Code(CXzUnpacker *p, Byte *dest, SizeT *destLen, { if (p->pos < XZ_SIG_SIZE && *src != XZ_SIG[p->pos]) return SZ_ERROR_NO_ARCHIVE; + if (p->decodeToStreamSignature) + return SZ_OK; p->buf[p->pos++] = *src++; (*srcLen)++; } @@ -808,20 +1154,32 @@ SRes XzUnpacker_Code(CXzUnpacker *p, Byte *dest, SizeT *destLen, else { RINOK(XzBlock_Parse(&p->block, p->buf)); + if (!XzBlock_AreSupportedFilters(&p->block)) + return SZ_ERROR_UNSUPPORTED; p->numTotalBlocks++; p->state = XZ_STATE_BLOCK; p->packSize = 0; p->unpackSize = 0; XzCheck_Init(&p->check, XzFlags_GetCheckType(p->streamFlags)); - RINOK(XzDec_Init(&p->decoder, &p->block)); + if (p->parseMode) + { + p->headerParsedOk = True; + return SZ_OK; + } + RINOK(XzDecMix_Init(&p->decoder, &p->block, p->outBuf, p->outBufSize)); } break; } case XZ_STATE_BLOCK_FOOTER: { - if (((p->packSize + p->alignPos) & 3) != 0) + if ((((unsigned)p->packSize + p->alignPos) & 3) != 0) { + if (srcRem == 0) + { + *status = CODER_STATUS_NEEDS_MORE_INPUT; + return SZ_OK; + } (*srcLen)++; p->alignPos++; if (*src++ != 0) @@ -833,6 +1191,11 @@ SRes XzUnpacker_Code(CXzUnpacker *p, Byte *dest, SizeT *destLen, UInt32 cur = checkSize - p->pos; if (cur != 0) { + if (srcRem == 0) + { + *status = CODER_STATUS_NEEDS_MORE_INPUT; + return SZ_OK; + } if (cur > srcRem) cur = (UInt32)srcRem; memcpy(p->buf + p->pos, src, cur); @@ -969,6 +1332,20 @@ SRes XzUnpacker_Code(CXzUnpacker *p, Byte *dest, SizeT *destLen, } +SRes XzUnpacker_CodeFull(CXzUnpacker *p, Byte *dest, SizeT *destLen, + const Byte *src, SizeT *srcLen, + ECoderFinishMode finishMode, ECoderStatus *status) +{ + XzUnpacker_Init(p); + XzUnpacker_SetOutBuf(p, dest, *destLen); + + return XzUnpacker_Code(p, + NULL, destLen, + src, srcLen, True, + finishMode, status); +} + + Bool XzUnpacker_IsBlockFinished(const CXzUnpacker *p) { return (p->state == XZ_STATE_BLOCK_HEADER) && (p->pos == 0); @@ -983,8 +1360,1414 @@ UInt64 XzUnpacker_GetExtraSize(const CXzUnpacker *p) { UInt64 num = 0; if (p->state == XZ_STATE_STREAM_PADDING) - num += p->padSize; + num = p->padSize; else if (p->state == XZ_STATE_STREAM_HEADER) - num += p->padSize + p->pos; + num = p->padSize + p->pos; return num; } + + + + + + + + + + + + + + + + + + + + + +#ifndef _7ZIP_ST +#include "MtDec.h" +#endif + + +void XzDecMtProps_Init(CXzDecMtProps *p) +{ + p->inBufSize_ST = 1 << 18; + p->outStep_ST = 1 << 20; + p->ignoreErrors = False; + + #ifndef _7ZIP_ST + p->numThreads = 1; + p->inBufSize_MT = 1 << 18; + p->memUseMax = sizeof(size_t) << 28; + #endif +} + + + +#ifndef _7ZIP_ST + +/* ---------- CXzDecMtThread ---------- */ + +typedef struct +{ + Byte *outBuf; + size_t outBufSize; + size_t outPreSize; + size_t inPreSize; + size_t inPreHeaderSize; + size_t blockPackSize_for_Index; // including block header and checksum. + size_t blockPackTotal; // including stream header, block header and checksum. + size_t inCodeSize; + size_t outCodeSize; + ECoderStatus status; + SRes codeRes; + Bool skipMode; + // Bool finishedWithMark; + EMtDecParseState parseState; + Bool parsing_Truncated; + Bool atBlockHeader; + CXzStreamFlags streamFlags; + // UInt64 numFinishedStreams + UInt64 numStreams; + UInt64 numTotalBlocks; + UInt64 numBlocks; + + Bool dec_created; + CXzUnpacker dec; + + Byte mtPad[1 << 7]; +} CXzDecMtThread; + +#endif + + +/* ---------- CXzDecMt ---------- */ + +typedef struct +{ + CAlignOffsetAlloc alignOffsetAlloc; + ISzAllocPtr allocMid; + + CXzDecMtProps props; + size_t unpackBlockMaxSize; + + ISeqInStream *inStream; + ISeqOutStream *outStream; + ICompressProgress *progress; + // CXzStatInfo *stat; + + Bool finishMode; + Bool outSize_Defined; + UInt64 outSize; + + UInt64 outProcessed; + UInt64 inProcessed; + UInt64 readProcessed; + Bool readWasFinished; + SRes readRes; + SRes writeRes; + + Byte *outBuf; + size_t outBufSize; + Byte *inBuf; + size_t inBufSize; + Bool dec_created; + CXzUnpacker dec; + + ECoderStatus status; + SRes codeRes; + + #ifndef _7ZIP_ST + Bool mainDecoderWasCalled; + // int statErrorDefined; + int finishedDecoderIndex; + + // global values that are used in Parse stage + CXzStreamFlags streamFlags; + // UInt64 numFinishedStreams + UInt64 numStreams; + UInt64 numTotalBlocks; + UInt64 numBlocks; + + // UInt64 numBadBlocks; + SRes mainErrorCode; + + Bool isBlockHeaderState_Parse; + Bool isBlockHeaderState_Write; + UInt64 outProcessed_Parse; + Bool parsing_Truncated; + + Bool mtc_WasConstructed; + CMtDec mtc; + CXzDecMtThread coders[MTDEC__THREADS_MAX]; + #endif + +} CXzDecMt; + + + +CXzDecMtHandle XzDecMt_Create(ISzAllocPtr alloc, ISzAllocPtr allocMid) +{ + CXzDecMt *p = (CXzDecMt *)ISzAlloc_Alloc(alloc, sizeof(CXzDecMt)); + if (!p) + return NULL; + + AlignOffsetAlloc_CreateVTable(&p->alignOffsetAlloc); + p->alignOffsetAlloc.baseAlloc = alloc; + p->alignOffsetAlloc.numAlignBits = 7; + p->alignOffsetAlloc.offset = 0; + + p->allocMid = allocMid; + + p->outBuf = NULL; + p->outBufSize = 0; + p->inBuf = NULL; + p->inBufSize = 0; + p->dec_created = False; + + p->unpackBlockMaxSize = 0; + + XzDecMtProps_Init(&p->props); + + #ifndef _7ZIP_ST + p->mtc_WasConstructed = False; + { + unsigned i; + for (i = 0; i < MTDEC__THREADS_MAX; i++) + { + CXzDecMtThread *coder = &p->coders[i]; + coder->dec_created = False; + coder->outBuf = NULL; + coder->outBufSize = 0; + } + } + #endif + + return p; +} + + +#ifndef _7ZIP_ST + +static void XzDecMt_FreeOutBufs(CXzDecMt *p) +{ + unsigned i; + for (i = 0; i < MTDEC__THREADS_MAX; i++) + { + CXzDecMtThread *coder = &p->coders[i]; + if (coder->outBuf) + { + ISzAlloc_Free(p->allocMid, coder->outBuf); + coder->outBuf = NULL; + coder->outBufSize = 0; + } + } + p->unpackBlockMaxSize = 0; +} + +#endif + + + +static void XzDecMt_FreeSt(CXzDecMt *p) +{ + if (p->dec_created) + { + XzUnpacker_Free(&p->dec); + p->dec_created = False; + } + + if (p->outBuf) + { + ISzAlloc_Free(p->allocMid, p->outBuf); + p->outBuf = NULL; + } + p->outBufSize = 0; + + if (p->inBuf) + { + ISzAlloc_Free(p->allocMid, p->inBuf); + p->inBuf = NULL; + } + p->inBufSize = 0; +} + + +void XzDecMt_Destroy(CXzDecMtHandle pp) +{ + CXzDecMt *p = (CXzDecMt *)pp; + + XzDecMt_FreeSt(p); + + #ifndef _7ZIP_ST + + if (p->mtc_WasConstructed) + { + MtDec_Destruct(&p->mtc); + p->mtc_WasConstructed = False; + } + { + unsigned i; + for (i = 0; i < MTDEC__THREADS_MAX; i++) + { + CXzDecMtThread *t = &p->coders[i]; + if (t->dec_created) + { + // we don't need to free dict here + XzUnpacker_Free(&t->dec); + t->dec_created = False; + } + } + } + XzDecMt_FreeOutBufs(p); + + #endif + + ISzAlloc_Free(p->alignOffsetAlloc.baseAlloc, pp); +} + + + +#ifndef _7ZIP_ST + +static void XzDecMt_Callback_Parse(void *obj, unsigned coderIndex, CMtDecCallbackInfo *cc) +{ + CXzDecMt *me = (CXzDecMt *)obj; + CXzDecMtThread *coder = &me->coders[coderIndex]; + size_t srcSize = cc->srcSize; + + cc->srcSize = 0; + cc->outPos = 0; + cc->state = MTDEC_PARSE_CONTINUE; + + cc->canCreateNewThread = True; + + if (cc->startCall) + { + coder->outPreSize = 0; + coder->inPreSize = 0; + coder->inPreHeaderSize = 0; + coder->parseState = MTDEC_PARSE_CONTINUE; + coder->parsing_Truncated = False; + coder->skipMode = False; + coder->codeRes = SZ_OK; + coder->status = CODER_STATUS_NOT_SPECIFIED; + coder->inCodeSize = 0; + coder->outCodeSize = 0; + + coder->numStreams = me->numStreams; + coder->numTotalBlocks = me->numTotalBlocks; + coder->numBlocks = me->numBlocks; + + if (!coder->dec_created) + { + XzUnpacker_Construct(&coder->dec, &me->alignOffsetAlloc.vt); + coder->dec_created = True; + } + + XzUnpacker_Init(&coder->dec); + + if (me->isBlockHeaderState_Parse) + { + coder->dec.streamFlags = me->streamFlags; + coder->atBlockHeader = True; + XzUnpacker_PrepareToRandomBlockDecoding(&coder->dec); + } + else + { + coder->atBlockHeader = False; + me->isBlockHeaderState_Parse = True; + } + + coder->dec.numStartedStreams = me->numStreams; + coder->dec.numTotalBlocks = me->numTotalBlocks; + coder->dec.numBlocks = me->numBlocks; + } + + while (!coder->skipMode) + { + ECoderStatus status; + SRes res; + size_t srcSize2 = srcSize; + size_t destSize = (size_t)0 - 1; + + coder->dec.parseMode = True; + coder->dec.headerParsedOk = False; + + PRF_STR_INT("Parse", srcSize2); + + res = XzUnpacker_Code(&coder->dec, + NULL, &destSize, + cc->src, &srcSize2, cc->srcFinished, + CODER_FINISH_END, &status); + + // PRF(printf(" res = %d, srcSize2 = %d", res, (unsigned)srcSize2)); + + coder->codeRes = res; + coder->status = status; + cc->srcSize += srcSize2; + srcSize -= srcSize2; + coder->inPreHeaderSize += srcSize2; + coder->inPreSize = coder->inPreHeaderSize; + + if (res != SZ_OK) + { + cc->state = + coder->parseState = MTDEC_PARSE_END; + /* + if (res == SZ_ERROR_MEM) + return res; + return SZ_OK; + */ + return; // res; + } + + if (coder->dec.headerParsedOk) + { + const CXzBlock *block = &coder->dec.block; + if (XzBlock_HasUnpackSize(block) + // && block->unpackSize <= me->props.outBlockMax + && XzBlock_HasPackSize(block)) + { + { + if (block->unpackSize * 2 * me->mtc.numStartedThreads > me->props.memUseMax) + { + cc->state = MTDEC_PARSE_OVERFLOW; + return; // SZ_OK; + } + } + { + UInt64 packSize = block->packSize; + UInt64 packSizeAligned = packSize + ((0 - (unsigned)packSize) & 3); + UInt32 checkSize = XzFlags_GetCheckSize(coder->dec.streamFlags); + UInt64 blockPackSum = coder->inPreSize + packSizeAligned + checkSize; + // if (blockPackSum <= me->props.inBlockMax) + // unpackBlockMaxSize + { + coder->blockPackSize_for_Index = (size_t)(coder->dec.blockHeaderSize + packSize + checkSize); + coder->blockPackTotal = (size_t)blockPackSum; + coder->outPreSize = (size_t)block->unpackSize; + coder->streamFlags = coder->dec.streamFlags; + me->streamFlags = coder->dec.streamFlags; + coder->skipMode = True; + break; + } + } + } + } + else + // if (coder->inPreSize <= me->props.inBlockMax) + { + if (!cc->srcFinished) + return; // SZ_OK; + cc->state = + coder->parseState = MTDEC_PARSE_END; + return; // SZ_OK; + } + cc->state = MTDEC_PARSE_OVERFLOW; + return; // SZ_OK; + } + + // ---------- skipMode ---------- + { + UInt64 rem = coder->blockPackTotal - coder->inPreSize; + size_t cur = srcSize; + if (cur > rem) + cur = (size_t)rem; + cc->srcSize += cur; + coder->inPreSize += cur; + srcSize -= cur; + + if (coder->inPreSize == coder->blockPackTotal) + { + if (srcSize == 0) + { + if (!cc->srcFinished) + return; // SZ_OK; + cc->state = MTDEC_PARSE_END; + } + else if ((cc->src)[cc->srcSize] == 0) // we check control byte of next block + cc->state = MTDEC_PARSE_END; + else + { + cc->state = MTDEC_PARSE_NEW; + + { + size_t blockMax = me->unpackBlockMaxSize; + if (blockMax < coder->outPreSize) + blockMax = coder->outPreSize; + { + UInt64 required = (UInt64)blockMax * (me->mtc.numStartedThreads + 1) * 2; + if (me->props.memUseMax < required) + cc->canCreateNewThread = False; + } + } + + if (me->outSize_Defined) + { + // next block can be zero size + const UInt64 rem2 = me->outSize - me->outProcessed_Parse; + if (rem2 < coder->outPreSize) + { + coder->parsing_Truncated = True; + cc->state = MTDEC_PARSE_END; + } + me->outProcessed_Parse += coder->outPreSize; + } + } + } + else if (cc->srcFinished) + cc->state = MTDEC_PARSE_END; + else + return; // SZ_OK; + + coder->parseState = cc->state; + cc->outPos = coder->outPreSize; + + me->numStreams = coder->dec.numStartedStreams; + me->numTotalBlocks = coder->dec.numTotalBlocks; + me->numBlocks = coder->dec.numBlocks + 1; + return; // SZ_OK; + } +} + + +static SRes XzDecMt_Callback_PreCode(void *pp, unsigned coderIndex) +{ + CXzDecMt *me = (CXzDecMt *)pp; + CXzDecMtThread *coder = &me->coders[coderIndex]; + Byte *dest; + + if (!coder->dec.headerParsedOk) + return SZ_OK; + + dest = coder->outBuf; + + if (!dest || coder->outBufSize < coder->outPreSize) + { + if (dest) + { + ISzAlloc_Free(me->allocMid, dest); + coder->outBuf = NULL; + coder->outBufSize = 0; + } + { + size_t outPreSize = coder->outPreSize; + if (outPreSize == 0) + outPreSize = 1; + dest = (Byte *)ISzAlloc_Alloc(me->allocMid, outPreSize); + } + if (!dest) + return SZ_ERROR_MEM; + coder->outBuf = dest; + coder->outBufSize = coder->outPreSize; + + if (coder->outBufSize > me->unpackBlockMaxSize) + me->unpackBlockMaxSize = coder->outBufSize; + } + + // return SZ_ERROR_MEM; + + XzUnpacker_SetOutBuf(&coder->dec, coder->outBuf, coder->outBufSize); + + { + SRes res = XzDecMix_Init(&coder->dec.decoder, &coder->dec.block, coder->outBuf, coder->outBufSize); + // res = SZ_ERROR_UNSUPPORTED; // to test + coder->codeRes = res; + if (res != SZ_OK) + { + // if (res == SZ_ERROR_MEM) return res; + if (me->props.ignoreErrors && res != SZ_ERROR_MEM) + return S_OK; + return res; + } + } + + return SZ_OK; +} + + +static SRes XzDecMt_Callback_Code(void *pp, unsigned coderIndex, + const Byte *src, size_t srcSize, int srcFinished, + // int finished, int blockFinished, + UInt64 *inCodePos, UInt64 *outCodePos, int *stop) +{ + CXzDecMt *me = (CXzDecMt *)pp; + CXzDecMtThread *coder = &me->coders[coderIndex]; + + *inCodePos = coder->inCodeSize; + *outCodePos = coder->outCodeSize; + *stop = True; + + if (coder->inCodeSize < coder->inPreHeaderSize) + { + UInt64 rem = coder->inPreHeaderSize - coder->inCodeSize; + size_t step = srcSize; + if (step > rem) + step = (size_t)rem; + src += step; + srcSize -= step; + coder->inCodeSize += step; + if (coder->inCodeSize < coder->inPreHeaderSize) + { + *stop = False; + return SZ_OK; + } + } + + if (!coder->dec.headerParsedOk) + return SZ_OK; + if (!coder->outBuf) + return SZ_OK; + + if (coder->codeRes == SZ_OK) + { + ECoderStatus status; + SRes res; + size_t srcProcessed = srcSize; + size_t outSizeCur = coder->outPreSize - coder->dec.outDataWritten; + + // PRF(printf("\nCallback_Code: Code %d %d\n", (unsigned)srcSize, (unsigned)outSizeCur)); + + res = XzUnpacker_Code(&coder->dec, + NULL, &outSizeCur, + src, &srcProcessed, srcFinished, + // coder->finishedWithMark ? CODER_FINISH_END : CODER_FINISH_ANY, + CODER_FINISH_END, + &status); + + // PRF(printf(" res = %d, srcSize2 = %d, outSizeCur = %d", res, (unsigned)srcProcessed, (unsigned)outSizeCur)); + + coder->codeRes = res; + coder->status = status; + coder->inCodeSize += srcProcessed; + coder->outCodeSize = coder->dec.outDataWritten; + *inCodePos = coder->inCodeSize; + *outCodePos = coder->outCodeSize; + + if (res == SZ_OK) + { + if (srcProcessed == srcSize) + *stop = False; + return SZ_OK; + } + } + + if (me->props.ignoreErrors && coder->codeRes != SZ_ERROR_MEM) + { + *inCodePos = coder->inPreSize; + *outCodePos = coder->outPreSize; + return S_OK; + } + return coder->codeRes; +} + + +#define XZDECMT_STREAM_WRITE_STEP (1 << 24) + +static SRes XzDecMt_Callback_Write(void *pp, unsigned coderIndex, + Bool needWriteToStream, + const Byte *src, size_t srcSize, + // int srcFinished, + Bool *needContinue, + Bool *canRecode) +{ + CXzDecMt *me = (CXzDecMt *)pp; + const CXzDecMtThread *coder = &me->coders[coderIndex]; + + // PRF(printf("\nWrite processed = %d srcSize = %d\n", (unsigned)me->mtc.inProcessed, (unsigned)srcSize)); + + *needContinue = False; + *canRecode = True; + + if (!needWriteToStream) + return SZ_OK; + + if (!coder->dec.headerParsedOk || !coder->outBuf) + { + if (me->finishedDecoderIndex < 0) + me->finishedDecoderIndex = coderIndex; + return SZ_OK; + } + + if (me->finishedDecoderIndex >= 0) + return SZ_OK; + + me->mtc.inProcessed += coder->inCodeSize; + + *canRecode = False; + + { + SRes res; + size_t size = coder->outCodeSize; + Byte *data = coder->outBuf; + + // we use in me->dec: sha, numBlocks, indexSize + + if (!me->isBlockHeaderState_Write) + { + XzUnpacker_PrepareToRandomBlockDecoding(&me->dec); + me->dec.decodeOnlyOneBlock = False; + me->dec.numStartedStreams = coder->dec.numStartedStreams; + me->dec.streamFlags = coder->streamFlags; + + me->isBlockHeaderState_Write = True; + } + + me->dec.numTotalBlocks = coder->dec.numTotalBlocks; + XzUnpacker_UpdateIndex(&me->dec, coder->blockPackSize_for_Index, coder->outPreSize); + + if (coder->outPreSize != size) + { + if (me->props.ignoreErrors) + { + memset(data + size, 0, coder->outPreSize - size); + size = coder->outPreSize; + } + // me->numBadBlocks++; + if (me->mainErrorCode == SZ_OK) + { + if ((int)coder->status == LZMA_STATUS_NEEDS_MORE_INPUT) + me->mainErrorCode = SZ_ERROR_INPUT_EOF; + else + me->mainErrorCode = SZ_ERROR_DATA; + } + } + + if (me->writeRes != SZ_OK) + return me->writeRes; + + res = SZ_OK; + { + if (me->outSize_Defined) + { + const UInt64 rem = me->outSize - me->outProcessed; + if (size > rem) + size = (SizeT)rem; + } + + for (;;) + { + size_t cur = size; + size_t written; + if (cur > XZDECMT_STREAM_WRITE_STEP) + cur = XZDECMT_STREAM_WRITE_STEP; + + written = ISeqOutStream_Write(me->outStream, data, cur); + + // PRF(printf("\nWritten ask = %d written = %d\n", (unsigned)cur, (unsigned)written)); + + me->outProcessed += written; + if (written != cur) + { + me->writeRes = SZ_ERROR_WRITE; + res = me->writeRes; + break; + } + data += cur; + size -= cur; + // PRF_STR_INT("Written size =", size); + if (size == 0) + break; + res = MtProgress_ProgressAdd(&me->mtc.mtProgress, 0, 0); + if (res != SZ_OK) + break; + } + } + + if (coder->codeRes != SZ_OK) + if (!me->props.ignoreErrors) + { + me->finishedDecoderIndex = coderIndex; + return res; + } + + RINOK(res); + + if (coder->inPreSize != coder->inCodeSize + || coder->blockPackTotal != coder->inCodeSize) + { + me->finishedDecoderIndex = coderIndex; + return SZ_OK; + } + + if (coder->parseState != MTDEC_PARSE_END) + { + *needContinue = True; + return SZ_OK; + } + } + + // (coder->state == MTDEC_PARSE_END) means that there are no other working threads + // so we can use mtc variables without lock + + PRF_STR_INT("Write MTDEC_PARSE_END", me->mtc.inProcessed); + + me->mtc.mtProgress.totalInSize = me->mtc.inProcessed; + { + CXzUnpacker *dec = &me->dec; + + PRF_STR_INT("PostSingle", srcSize); + + { + size_t srcProcessed = srcSize; + ECoderStatus status; + size_t outSizeCur = 0; + SRes res; + + // dec->decodeOnlyOneBlock = False; + dec->decodeToStreamSignature = True; + + me->mainDecoderWasCalled = True; + + if (coder->parsing_Truncated) + { + me->parsing_Truncated = True; + return SZ_OK; + } + + res = XzUnpacker_Code(dec, + NULL, &outSizeCur, + src, &srcProcessed, + me->mtc.readWasFinished, // srcFinished + CODER_FINISH_END, // CODER_FINISH_ANY, + &status); + + me->status = status; + me->codeRes = res; + + me->mtc.inProcessed += srcProcessed; + me->mtc.mtProgress.totalInSize = me->mtc.inProcessed; + + if (res != SZ_OK) + { + return S_OK; + // return res; + } + + if (dec->state == XZ_STATE_STREAM_HEADER) + { + *needContinue = True; + me->isBlockHeaderState_Parse = False; + me->isBlockHeaderState_Write = False; + { + Byte *crossBuf = MtDec_GetCrossBuff(&me->mtc); + if (!crossBuf) + return SZ_ERROR_MEM; + memcpy(crossBuf, src + srcProcessed, srcSize - srcProcessed); + } + me->mtc.crossStart = 0; + me->mtc.crossEnd = srcSize - srcProcessed; + return SZ_OK; + } + + if (status != CODER_STATUS_NEEDS_MORE_INPUT) + { + return E_FAIL; + } + + if (me->mtc.readWasFinished) + { + return SZ_OK; + } + } + + { + size_t inPos; + size_t inLim; + const Byte *inData; + UInt64 inProgressPrev = me->mtc.inProcessed; + + // XzDecMt_Prepare_InBuf_ST(p); + Byte *crossBuf = MtDec_GetCrossBuff(&me->mtc); + if (!crossBuf) + return SZ_ERROR_MEM; + + inPos = 0; + inLim = 0; + // outProcessed = 0; + + inData = crossBuf; + + for (;;) + { + SizeT inProcessed; + SizeT outProcessed; + ECoderStatus status; + SRes res; + + if (inPos == inLim) + { + if (!me->mtc.readWasFinished) + { + inPos = 0; + inLim = me->mtc.inBufSize; + me->mtc.readRes = ISeqInStream_Read(me->inStream, (void *)inData, &inLim); + me->mtc.readProcessed += inLim; + if (inLim == 0 || me->mtc.readRes != SZ_OK) + me->mtc.readWasFinished = True; + } + } + + inProcessed = inLim - inPos; + outProcessed = 0; + + res = XzUnpacker_Code(dec, + NULL, &outProcessed, + inData + inPos, &inProcessed, + (inProcessed == 0), // srcFinished + CODER_FINISH_END, &status); + + me->codeRes = res; + me->status = status; + inPos += inProcessed; + me->mtc.inProcessed += inProcessed; + me->mtc.mtProgress.totalInSize = me->mtc.inProcessed; + + if (res != SZ_OK) + { + return S_OK; + // return res; + } + + if (dec->state == XZ_STATE_STREAM_HEADER) + { + *needContinue = True; + me->mtc.crossStart = inPos; + me->mtc.crossEnd = inLim; + me->isBlockHeaderState_Parse = False; + me->isBlockHeaderState_Write = False; + return SZ_OK; + } + + if (status != CODER_STATUS_NEEDS_MORE_INPUT) + return E_FAIL; + + if (me->mtc.progress) + { + UInt64 inDelta = me->mtc.inProcessed - inProgressPrev; + if (inDelta >= (1 << 22)) + { + RINOK(MtProgress_Progress_ST(&me->mtc.mtProgress)); + inProgressPrev = me->mtc.inProcessed; + } + } + if (me->mtc.readWasFinished) + return SZ_OK; + } + } + } +} + + +#endif + + + +void XzStatInfo_Clear(CXzStatInfo *p) +{ + p->InSize = 0; + p->OutSize = 0; + + p->NumStreams = 0; + p->NumBlocks = 0; + + p->UnpackSize_Defined = False; + + p->NumStreams_Defined = False; + p->NumBlocks_Defined = False; + + // p->IsArc = False; + // p->UnexpectedEnd = False; + // p->Unsupported = False; + // p->HeadersError = False; + // p->DataError = False; + // p->CrcError = False; + + p->DataAfterEnd = False; + p->DecodingTruncated = False; + + p->DecodeRes = SZ_OK; + p->ReadRes = SZ_OK; + p->ProgressRes = SZ_OK; + + p->CombinedRes = SZ_OK; + p->CombinedRes_Type = SZ_OK; +} + + + + +static SRes XzDecMt_Decode_ST(CXzDecMt *p + #ifndef _7ZIP_ST + , Bool tMode + #endif + , CXzStatInfo *stat) +{ + size_t outPos; + size_t inPos, inLim; + const Byte *inData; + UInt64 inPrev, outPrev; + + CXzUnpacker *dec; + + #ifndef _7ZIP_ST + if (tMode) + { + XzDecMt_FreeOutBufs(p); + tMode = MtDec_PrepareRead(&p->mtc); + } + #endif + + if (!p->outBuf || p->outBufSize != p->props.outStep_ST) + { + ISzAlloc_Free(p->allocMid, p->outBuf); + p->outBufSize = 0; + p->outBuf = (Byte *)ISzAlloc_Alloc(p->allocMid, p->props.outStep_ST); + if (!p->outBuf) + return SZ_ERROR_MEM; + p->outBufSize = p->props.outStep_ST; + } + + if (!p->inBuf || p->inBufSize != p->props.inBufSize_ST) + { + ISzAlloc_Free(p->allocMid, p->inBuf); + p->inBufSize = 0; + p->inBuf = (Byte *)ISzAlloc_Alloc(p->allocMid, p->props.inBufSize_ST); + if (!p->inBuf) + return SZ_ERROR_MEM; + p->inBufSize = p->props.inBufSize_ST; + } + + dec = &p->dec; + dec->decodeToStreamSignature = False; + // dec->decodeOnlyOneBlock = False; + + XzUnpacker_SetOutBuf(dec, NULL, 0); + + inPrev = p->inProcessed; + outPrev = p->outProcessed; + + inPos = 0; + inLim = 0; + inData = NULL; + outPos = 0; + + for (;;) + { + SizeT outSize; + Bool finished; + ECoderFinishMode finishMode; + SizeT inProcessed; + ECoderStatus status; + SRes res; + + SizeT outProcessed; + + + + if (inPos == inLim) + { + #ifndef _7ZIP_ST + if (tMode) + { + inData = MtDec_Read(&p->mtc, &inLim); + inPos = 0; + if (inData) + continue; + tMode = False; + inLim = 0; + } + #endif + + if (!p->readWasFinished) + { + inPos = 0; + inLim = p->inBufSize; + inData = p->inBuf; + p->readRes = ISeqInStream_Read(p->inStream, (void *)inData, &inLim); + p->readProcessed += inLim; + if (inLim == 0 || p->readRes != SZ_OK) + p->readWasFinished = True; + } + } + + outSize = p->props.outStep_ST - outPos; + + finishMode = CODER_FINISH_ANY; + if (p->outSize_Defined) + { + const UInt64 rem = p->outSize - p->outProcessed; + if (outSize >= rem) + { + outSize = (SizeT)rem; + if (p->finishMode) + finishMode = CODER_FINISH_END; + } + } + + inProcessed = inLim - inPos; + outProcessed = outSize; + + res = XzUnpacker_Code(dec, p->outBuf + outPos, &outProcessed, + inData + inPos, &inProcessed, + (inPos == inLim), // srcFinished + finishMode, &status); + + p->codeRes = res; + p->status = status; + + inPos += inProcessed; + outPos += outProcessed; + p->inProcessed += inProcessed; + p->outProcessed += outProcessed; + + finished = ((inProcessed == 0 && outProcessed == 0) || res != SZ_OK); + + if (finished || outProcessed >= outSize) + if (outPos != 0) + { + size_t written = ISeqOutStream_Write(p->outStream, p->outBuf, outPos); + p->outProcessed += written; + if (written != outPos) + { + stat->CombinedRes_Type = SZ_ERROR_WRITE; + return SZ_ERROR_WRITE; + } + outPos = 0; + } + + if (p->progress && res == SZ_OK) + { + UInt64 inDelta = p->inProcessed - inPrev; + UInt64 outDelta = p->outProcessed - outPrev; + if (inDelta >= (1 << 22) || outDelta >= (1 << 22)) + { + res = ICompressProgress_Progress(p->progress, p->inProcessed, p->outProcessed); + if (res != SZ_OK) + { + stat->CombinedRes_Type = SZ_ERROR_PROGRESS; + stat->ProgressRes = res; + return res; + } + inPrev = p->inProcessed; + outPrev = p->outProcessed; + } + } + + if (finished) + return res; + } +} + +static SRes XzStatInfo_SetStat(const CXzUnpacker *dec, + int finishMode, + UInt64 readProcessed, UInt64 inProcessed, + SRes res, ECoderStatus status, + Bool decodingTruncated, + CXzStatInfo *stat) +{ + UInt64 extraSize; + + stat->DecodingTruncated = (Byte)(decodingTruncated ? 1 : 0); + stat->InSize = inProcessed; + stat->NumStreams = dec->numStartedStreams; + stat->NumBlocks = dec->numTotalBlocks; + + stat->UnpackSize_Defined = True; + stat->NumStreams_Defined = True; + stat->NumBlocks_Defined = True; + + extraSize = XzUnpacker_GetExtraSize(dec); + + if (res == SZ_OK) + { + if (status == CODER_STATUS_NEEDS_MORE_INPUT) + { + // CODER_STATUS_NEEDS_MORE_INPUT is expected status for correct xz streams + extraSize = 0; + if (!XzUnpacker_IsStreamWasFinished(dec)) + res = SZ_ERROR_INPUT_EOF; + } + else if (!decodingTruncated || finishMode) // (status == CODER_STATUS_NOT_FINISHED) + res = SZ_ERROR_DATA; + } + else if (res == SZ_ERROR_NO_ARCHIVE) + { + /* + SZ_ERROR_NO_ARCHIVE is possible for 2 states: + XZ_STATE_STREAM_HEADER - if bad signature or bad CRC + XZ_STATE_STREAM_PADDING - if non-zero padding data + extraSize / inProcessed don't include "bad" byte + */ + if (inProcessed != extraSize) // if good streams before error + if (extraSize != 0 || readProcessed != inProcessed) + { + stat->DataAfterEnd = True; + // there is some good xz stream before. So we set SZ_OK + res = SZ_OK; + } + } + + stat->DecodeRes = res; + + stat->InSize -= extraSize; + return res; +} + + +SRes XzDecMt_Decode(CXzDecMtHandle pp, + const CXzDecMtProps *props, + const UInt64 *outDataSize, int finishMode, + ISeqOutStream *outStream, + // Byte *outBuf, size_t *outBufSize, + ISeqInStream *inStream, + // const Byte *inData, size_t inDataSize, + CXzStatInfo *stat, + int *isMT, + ICompressProgress *progress) +{ + CXzDecMt *p = (CXzDecMt *)pp; + #ifndef _7ZIP_ST + Bool tMode; + #endif + + XzStatInfo_Clear(stat); + + p->props = *props; + + p->inStream = inStream; + p->outStream = outStream; + p->progress = progress; + // p->stat = stat; + + p->outSize = 0; + p->outSize_Defined = False; + if (outDataSize) + { + p->outSize_Defined = True; + p->outSize = *outDataSize; + } + + p->finishMode = finishMode; + + // p->outSize = 457; p->outSize_Defined = True; p->finishMode = False; // for test + + p->writeRes = SZ_OK; + p->outProcessed = 0; + p->inProcessed = 0; + p->readProcessed = 0; + p->readWasFinished = False; + + p->codeRes = 0; + p->status = CODER_STATUS_NOT_SPECIFIED; + + if (!p->dec_created) + { + XzUnpacker_Construct(&p->dec, &p->alignOffsetAlloc.vt); + p->dec_created = True; + } + XzUnpacker_Init(&p->dec); + + + *isMT = False; + + /* + p->outBuf = NULL; + p->outBufSize = 0; + if (!outStream) + { + p->outBuf = outBuf; + p->outBufSize = *outBufSize; + *outBufSize = 0; + } + */ + + + #ifndef _7ZIP_ST + + p->isBlockHeaderState_Parse = False; + p->isBlockHeaderState_Write = False; + // p->numBadBlocks = 0; + p->mainErrorCode = SZ_OK; + p->mainDecoderWasCalled = False; + + tMode = False; + + if (p->props.numThreads > 1) + { + IMtDecCallback vt; + + XzDecMt_FreeSt(p); + + p->outProcessed_Parse = 0; + p->parsing_Truncated = False; + + p->numStreams = 0; + p->numTotalBlocks = 0; + p->numBlocks = 0; + p->finishedDecoderIndex = -1; + + if (!p->mtc_WasConstructed) + { + p->mtc_WasConstructed = True; + MtDec_Construct(&p->mtc); + } + + p->mtc.mtCallback = &vt; + p->mtc.mtCallbackObject = p; + + p->mtc.progress = progress; + p->mtc.inStream = inStream; + p->mtc.alloc = &p->alignOffsetAlloc.vt; + // p->mtc.inData = inData; + // p->mtc.inDataSize = inDataSize; + p->mtc.inBufSize = p->props.inBufSize_MT; + // p->mtc.inBlockMax = p->props.inBlockMax; + p->mtc.numThreadsMax = p->props.numThreads; + + *isMT = True; + + vt.Parse = XzDecMt_Callback_Parse; + vt.PreCode = XzDecMt_Callback_PreCode; + vt.Code = XzDecMt_Callback_Code; + vt.Write = XzDecMt_Callback_Write; + + { + Bool needContinue; + + SRes res = MtDec_Code(&p->mtc); + + stat->InSize = p->mtc.inProcessed; + + p->inProcessed = p->mtc.inProcessed; + p->readRes = p->mtc.readRes; + p->readWasFinished = p->mtc.readWasFinished; + p->readProcessed = p->mtc.readProcessed; + + tMode = True; + needContinue = False; + + if (res == SZ_OK) + { + if (p->mtc.mtProgress.res != SZ_OK) + { + res = p->mtc.mtProgress.res; + stat->ProgressRes = res; + stat->CombinedRes_Type = SZ_ERROR_PROGRESS; + } + else + needContinue = p->mtc.needContinue; + } + + if (!needContinue) + { + SRes codeRes; + Bool truncated = False; + ECoderStatus status; + CXzUnpacker *dec; + + stat->OutSize = p->outProcessed; + + if (p->finishedDecoderIndex >= 0) + { + CXzDecMtThread *coder = &p->coders[(unsigned)p->finishedDecoderIndex]; + codeRes = coder->codeRes; + dec = &coder->dec; + status = coder->status; + } + else if (p->mainDecoderWasCalled) + { + codeRes = p->codeRes; + dec = &p->dec; + status = p->status; + truncated = p->parsing_Truncated; + } + else + return E_FAIL; + + XzStatInfo_SetStat(dec, p->finishMode, + p->mtc.readProcessed, p->mtc.inProcessed, + codeRes, status, + truncated, + stat); + + if (res == SZ_OK) + { + if (p->writeRes != SZ_OK) + { + res = p->writeRes; + stat->CombinedRes_Type = SZ_ERROR_WRITE; + } + else if (p->mtc.readRes != SZ_OK && p->mtc.inProcessed == p->mtc.readProcessed) + { + res = p->mtc.readRes; + stat->ReadRes = res; + stat->CombinedRes_Type = SZ_ERROR_READ; + } + else if (p->mainErrorCode != SZ_OK) + { + res = p->mainErrorCode; + } + } + + stat->CombinedRes = res; + if (stat->CombinedRes_Type == SZ_OK) + stat->CombinedRes_Type = res; + return res; + } + + PRF_STR("----- decoding ST -----"); + } + } + + #endif + + + *isMT = False; + + { + SRes res = XzDecMt_Decode_ST(p + #ifndef _7ZIP_ST + , tMode + #endif + , stat + ); + + XzStatInfo_SetStat(&p->dec, + p->finishMode, + p->readProcessed, p->inProcessed, + p->codeRes, p->status, + False, // truncated + stat); + + if (res == SZ_OK) + { + /* + if (p->writeRes != SZ_OK) + { + res = p->writeRes; + stat->CombinedRes_Type = SZ_ERROR_WRITE; + } + else + */ + if (p->readRes != SZ_OK && p->inProcessed == p->readProcessed) + { + res = p->readRes; + stat->ReadRes = res; + stat->CombinedRes_Type = SZ_ERROR_READ; + } + #ifndef _7ZIP_ST + else if (p->mainErrorCode != SZ_OK) + res = p->mainErrorCode; + #endif + } + + stat->CombinedRes = res; + if (stat->CombinedRes_Type == SZ_OK) + stat->CombinedRes_Type = res; + return res; + } +} diff --git a/C/XzEnc.c b/C/XzEnc.c index 6abde77d..432cbfe7 100644 --- a/C/XzEnc.c +++ b/C/XzEnc.c @@ -1,5 +1,5 @@ /* XzEnc.c -- Xz Encode -2017-08-25 : Igor Pavlov : Public domain */ +2018-04-28 : Igor Pavlov : Public domain */ #include "Precomp.h" @@ -136,7 +136,7 @@ static void XzEncIndex_Free(CXzEncIndex *p, ISzAllocPtr alloc) static SRes XzEncIndex_ReAlloc(CXzEncIndex *p, size_t newSize, ISzAllocPtr alloc) { - Byte *blocks = ISzAlloc_Alloc(alloc, newSize); + Byte *blocks = (Byte *)ISzAlloc_Alloc(alloc, newSize); if (!blocks) return SZ_ERROR_MEM; if (p->size != 0) @@ -329,7 +329,7 @@ static SRes SeqInFilter_Init(CSeqInFilter *p, const CXzFilter *props, ISzAllocPt { if (!p->buf) { - p->buf = ISzAlloc_Alloc(alloc, FILTER_BUF_SIZE); + p->buf = (Byte *)ISzAlloc_Alloc(alloc, FILTER_BUF_SIZE); if (!p->buf) return SZ_ERROR_MEM; } @@ -362,13 +362,16 @@ static SRes SeqInFilter_Read(const ISeqInStream *pp, void *data, size_t *size) } { SizeT srcLen = p->endPos - p->curPos; - int wasFinished; + ECoderStatus status; SRes res; *size = sizeOriginal; - res = p->StateCoder.Code(p->StateCoder.p, data, size, p->buf + p->curPos, &srcLen, - p->srcWasFinished, CODER_FINISH_ANY, &wasFinished); + res = p->StateCoder.Code2(p->StateCoder.p, + data, size, + p->buf + p->curPos, &srcLen, + p->srcWasFinished, CODER_FINISH_ANY, + &status); p->curPos += srcLen; - if (*size != 0 || srcLen == 0 || res != 0) + if (*size != 0 || srcLen == 0 || res != SZ_OK) return res; } } @@ -377,11 +380,17 @@ static SRes SeqInFilter_Read(const ISeqInStream *pp, void *data, size_t *size) static void SeqInFilter_Construct(CSeqInFilter *p) { p->buf = NULL; + p->StateCoder.p = NULL; p->p.Read = SeqInFilter_Read; } static void SeqInFilter_Free(CSeqInFilter *p, ISzAllocPtr alloc) { + if (p->StateCoder.p) + { + p->StateCoder.Free(p->StateCoder.p, alloc); + p->StateCoder.p = NULL; + } if (p->buf) { ISzAlloc_Free(alloc, p->buf); @@ -1055,7 +1064,7 @@ static SRes XzEnc_MtCallback_Code(void *pp, unsigned coderIndex, unsigned outBuf if (!dest) { - dest = ISzAlloc_Alloc(me->alloc, me->outBufSize); + dest = (Byte *)ISzAlloc_Alloc(me->alloc, me->outBufSize); if (!dest) return SZ_ERROR_MEM; me->outBufs[outBufIndex] = dest; @@ -1063,7 +1072,7 @@ static SRes XzEnc_MtCallback_Code(void *pp, unsigned coderIndex, unsigned outBuf MtProgressThunk_CreateVTable(&progressThunk); progressThunk.mtProgress = &me->mtCoder.mtProgress; - progressThunk.index = coderIndex; + MtProgressThunk_Init(&progressThunk); { CXzEncBlockInfo blockSizes; @@ -1224,7 +1233,7 @@ SRes XzEnc_Encode(CXzEncHandle pp, ISeqOutStream *outStream, ISeqInStream *inStr if (!p->outBufs[0] || t2 != p->outBufSize) { XzEnc_FreeOutBufs(p); - p->outBufs[0] = ISzAlloc_Alloc(p->alloc, t2); + p->outBufs[0] = (Byte *)ISzAlloc_Alloc(p->alloc, t2); if (!p->outBufs[0]) return SZ_ERROR_MEM; p->outBufSize = t2; @@ -1245,6 +1254,8 @@ SRes XzEnc_Encode(CXzEncHandle pp, ISeqOutStream *outStream, ISeqInStream *inStr && props->reduceSize >= progress2.inOffset) rem = props->reduceSize - progress2.inOffset; */ + + blockSizes.headerSize = 0; // for GCC RINOK(Xz_CompressBlock( &p->lzmaf_Items[0], diff --git a/C/XzIn.c b/C/XzIn.c index 30054c4a..42da1dec 100644 --- a/C/XzIn.c +++ b/C/XzIn.c @@ -1,5 +1,5 @@ /* XzIn.c - Xz input -2017-05-11 : Igor Pavlov : Public domain */ +2018-02-02 : Igor Pavlov : Public domain */ #include "Precomp.h" @@ -103,7 +103,7 @@ static SRes Xz_ReadIndex2(CXzStream *p, const Byte *buf, size_t size, ISzAllocPt { size_t i; p->numBlocks = numBlocks; - p->blocks = ISzAlloc_Alloc(alloc, sizeof(CXzBlockSizes) * numBlocks); + p->blocks = (CXzBlockSizes *)ISzAlloc_Alloc(alloc, sizeof(CXzBlockSizes) * numBlocks); if (!p->blocks) return SZ_ERROR_MEM; for (i = 0; i < numBlocks; i++) @@ -131,7 +131,7 @@ static SRes Xz_ReadIndex(CXzStream *p, ILookInStream *stream, UInt64 indexSize, size = (size_t)indexSize; if (size != indexSize) return SZ_ERROR_UNSUPPORTED; - buf = ISzAlloc_Alloc(alloc, size); + buf = (Byte *)ISzAlloc_Alloc(alloc, size); if (!buf) return SZ_ERROR_MEM; res = LookInStream_Read2(stream, buf, size, SZ_ERROR_UNSUPPORTED); diff --git a/CPP/7zip/Archive/7z/7zCompressionMode.h b/CPP/7zip/Archive/7z/7zCompressionMode.h index 80e41545..23600171 100644 --- a/CPP/7zip/Archive/7z/7zCompressionMode.h +++ b/CPP/7zip/Archive/7z/7zCompressionMode.h @@ -13,7 +13,9 @@ struct CMethodFull: public CMethodProps { CMethodId Id; UInt32 NumStreams; + int CodecIndex; + CMethodFull(): CodecIndex(-1) {} bool IsSimpleCoder() const { return NumStreams == 1; } }; diff --git a/CPP/7zip/Archive/7z/7zDecode.cpp b/CPP/7zip/Archive/7z/7zDecode.cpp index 59e0d8b8..2705ecb9 100644 --- a/CPP/7zip/Archive/7z/7zDecode.cpp +++ b/CPP/7zip/Archive/7z/7zDecode.cpp @@ -236,8 +236,8 @@ HRESULT CDecoder::Decode( _7Z_DECODER_CRYPRO_VARS_DECL - #if !defined(_7ZIP_ST) && !defined(_SFX) - , bool mtMode, UInt32 numThreads + #if !defined(_7ZIP_ST) + , bool mtMode, UInt32 numThreads, UInt64 memUsage #endif ) { @@ -312,7 +312,7 @@ HRESULT CDecoder::Decode( #endif CCreatedCoder cod; - RINOK(CreateCoder( + RINOK(CreateCoder_Id( EXTERNAL_CODECS_LOC_VARS coderInfo.MethodID, false, cod)); @@ -355,11 +355,39 @@ HRESULT CDecoder::Decode( unsigned i; + bool mt_wasUsed = false; + for (i = 0; i < folderInfo.Coders.Size(); i++) { const CCoderInfo &coderInfo = folderInfo.Coders[i]; IUnknown *decoder = _mixer->GetCoder(i).GetUnknown(); + #if !defined(_7ZIP_ST) + if (!mt_wasUsed) + { + if (mtMode) + { + CMyComPtr setCoderMt; + decoder->QueryInterface(IID_ICompressSetCoderMt, (void **)&setCoderMt); + if (setCoderMt) + { + mt_wasUsed = true; + RINOK(setCoderMt->SetNumberOfThreads(numThreads)); + } + } + // if (memUsage != 0) + { + CMyComPtr setMemLimit; + decoder->QueryInterface(IID_ICompressSetMemLimit, (void **)&setMemLimit); + if (setMemLimit) + { + mt_wasUsed = true; + RINOK(setMemLimit->SetMemLimit(memUsage)); + } + } + } + #endif + { CMyComPtr setDecoderProperties; decoder->QueryInterface(IID_ICompressSetDecoderProperties2, (void **)&setDecoderProperties); @@ -376,18 +404,6 @@ HRESULT CDecoder::Decode( } } - #if !defined(_7ZIP_ST) && !defined(_SFX) - if (mtMode) - { - CMyComPtr setCoderMt; - decoder->QueryInterface(IID_ICompressSetCoderMt, (void **)&setCoderMt); - if (setCoderMt) - { - RINOK(setCoderMt->SetNumberOfThreads(numThreads)); - } - } - #endif - #ifndef _NO_CRYPTO { CMyComPtr cryptoSetPassword; diff --git a/CPP/7zip/Archive/7z/7zDecode.h b/CPP/7zip/Archive/7z/7zDecode.h index bca794b8..944f8a31 100644 --- a/CPP/7zip/Archive/7z/7zDecode.h +++ b/CPP/7zip/Archive/7z/7zDecode.h @@ -59,8 +59,8 @@ public: _7Z_DECODER_CRYPRO_VARS_DECL - #if !defined(_7ZIP_ST) && !defined(_SFX) - , bool mtMode, UInt32 numThreads + #if !defined(_7ZIP_ST) + , bool mtMode, UInt32 numThreads, UInt64 memUsage #endif ); }; diff --git a/CPP/7zip/Archive/7z/7zEncode.cpp b/CPP/7zip/Archive/7z/7zEncode.cpp index 0222831c..4c0d2214 100644 --- a/CPP/7zip/Archive/7z/7zEncode.cpp +++ b/CPP/7zip/Archive/7z/7zEncode.cpp @@ -154,9 +154,18 @@ HRESULT CEncoder::CreateMixerCoder( CCreatedCoder cod; - RINOK(CreateCoder( + if (methodFull.CodecIndex >= 0) + { + RINOK(CreateCoder_Index( + EXTERNAL_CODECS_LOC_VARS + methodFull.CodecIndex, true, cod)); + } + else + { + RINOK(CreateCoder_Id( EXTERNAL_CODECS_LOC_VARS methodFull.Id, true, cod)); + } if (cod.NumStreams != methodFull.NumStreams) return E_FAIL; diff --git a/CPP/7zip/Archive/7z/7zExtract.cpp b/CPP/7zip/Archive/7z/7zExtract.cpp index ef94d3c5..075644ff 100644 --- a/CPP/7zip/Archive/7z/7zExtract.cpp +++ b/CPP/7zip/Archive/7z/7zExtract.cpp @@ -152,6 +152,12 @@ STDMETHODIMP CFolderOutStream::Write(const void *data, UInt32 size, UInt32 *proc if (_fileIsOpen) { UInt32 cur = (size < _rem ? size : (UInt32)_rem); + if (_calcCrc) + { + const UInt32 k_Step = (UInt32)1 << 20; + if (cur > k_Step) + cur = k_Step; + } HRESULT result = S_OK; if (_stream) result = _stream->Write(data, cur, &cur); @@ -363,8 +369,8 @@ STDMETHODIMP CHandler::Extract(const UInt32 *indices, UInt32 numItems, , dataAfterEnd_Error _7Z_DECODER_CRYPRO_VARS - #if !defined(_7ZIP_ST) && !defined(_SFX) - , true, _numThreads + #if !defined(_7ZIP_ST) + , true, _numThreads, _memUsage #endif ); diff --git a/CPP/7zip/Archive/7z/7zHandler.cpp b/CPP/7zip/Archive/7z/7zHandler.cpp index 9267f66f..4ff87f76 100644 --- a/CPP/7zip/Archive/7z/7zHandler.cpp +++ b/CPP/7zip/Archive/7z/7zHandler.cpp @@ -40,7 +40,6 @@ CHandler::CHandler() _crcSize = 4; #ifdef __7Z_SET_PROPERTIES - _numThreads = NSystem::GetNumberOfProcessors(); _useMultiThreadMixer = true; #endif @@ -823,8 +822,8 @@ STDMETHODIMP CHandler::Close() STDMETHODIMP CHandler::SetProperties(const wchar_t * const *names, const PROPVARIANT *values, UInt32 numProps) { COM_TRY_BEGIN - const UInt32 numProcessors = NSystem::GetNumberOfProcessors(); - _numThreads = numProcessors; + + InitCommon(); _useMultiThreadMixer = true; for (UInt32 i = 0; i < numProps; i++) @@ -843,13 +842,15 @@ STDMETHODIMP CHandler::SetProperties(const wchar_t * const *names, const PROPVAR RINOK(PROPVARIANT_to_bool(value, _useMultiThreadMixer)); continue; } - if (name.IsPrefixedBy_Ascii_NoCase("mt")) { - RINOK(ParseMtProp(name.Ptr(2), value, numProcessors, _numThreads)); - continue; + HRESULT hres; + if (SetCommonProperty(name, value, hres)) + { + RINOK(hres); + continue; + } } - else - return E_INVALIDARG; + return E_INVALIDARG; } } return S_OK; diff --git a/CPP/7zip/Archive/7z/7zHandler.h b/CPP/7zip/Archive/7z/7zHandler.h index 33c240a9..7d5a5f00 100644 --- a/CPP/7zip/Archive/7z/7zHandler.h +++ b/CPP/7zip/Archive/7z/7zHandler.h @@ -8,16 +8,6 @@ #include "../../Common/CreateCoder.h" -#ifndef EXTRACT_ONLY -#include "../Common/HandlerOut.h" -#endif - -#include "7zCompressionMode.h" -#include "7zIn.h" - -namespace NArchive { -namespace N7z { - #ifndef __7Z_SET_PROPERTIES #ifdef EXTRACT_ONLY @@ -30,6 +20,16 @@ namespace N7z { #endif +// #ifdef __7Z_SET_PROPERTIES +#include "../Common/HandlerOut.h" +// #endif + +#include "7zCompressionMode.h" +#include "7zIn.h" + +namespace NArchive { +namespace N7z { + #ifndef EXTRACT_ONLY @@ -38,8 +38,6 @@ class COutHandler: public CMultiMethodProps HRESULT SetSolidFromString(const UString &s); HRESULT SetSolidFromPROPVARIANT(const PROPVARIANT &value); public: - bool _removeSfxBlock; - UInt64 _numSolidFiles; UInt64 _numSolidBytes; bool _numSolidBytesDefined; @@ -58,6 +56,8 @@ public: bool _useMultiThreadMixer; + bool _removeSfxBlock; + // bool _volumeMode; void InitSolidFiles() { _numSolidFiles = (UInt64)(Int64)(-1); } @@ -70,9 +70,10 @@ public: _numSolidBytesDefined = false; } + void InitProps7z(); void InitProps(); - COutHandler() { InitProps(); } + COutHandler() { InitProps7z(); } HRESULT SetProperty(const wchar_t *name, const PROPVARIANT &value); }; @@ -82,16 +83,23 @@ public: class CHandler: public IInArchive, public IArchiveGetRawProps, + #ifdef __7Z_SET_PROPERTIES public ISetProperties, #endif + #ifndef EXTRACT_ONLY public IOutArchive, #endif + PUBLIC_ISetCompressCodecsInfo - public CMyUnknownImp + + public CMyUnknownImp, + #ifndef EXTRACT_ONLY - , public COutHandler + public COutHandler + #else + public CCommonMethodProps #endif { public: @@ -135,7 +143,6 @@ private: #ifdef EXTRACT_ONLY #ifdef __7Z_SET_PROPERTIES - UInt32 _numThreads; bool _useMultiThreadMixer; #endif diff --git a/CPP/7zip/Archive/7z/7zHandlerOut.cpp b/CPP/7zip/Archive/7z/7zHandlerOut.cpp index d4aa4764..bfdf0943 100644 --- a/CPP/7zip/Archive/7z/7zHandlerOut.cpp +++ b/CPP/7zip/Archive/7z/7zHandlerOut.cpp @@ -13,6 +13,8 @@ #include "7zOut.h" #include "7zUpdate.h" +#ifndef EXTRACT_ONLY + using namespace NWindows; namespace NArchive { @@ -41,9 +43,11 @@ STDMETHODIMP CHandler::GetFileTimeType(UInt32 *type) HRESULT CHandler::PropsMethod_To_FullMethod(CMethodFull &dest, const COneMethodInfo &m) { - if (!FindMethod( + dest.CodecIndex = FindMethod_Index( EXTERNAL_CODECS_VARS - m.MethodName, dest.Id, dest.NumStreams)) + m.MethodName, true, + dest.Id, dest.NumStreams); + if (dest.CodecIndex < 0) return E_INVALIDARG; (CProps &)dest = (CProps &)m; return S_OK; @@ -700,10 +704,8 @@ static HRESULT ParseBond(UString &srcString, UInt32 &coder, UInt32 &stream) return S_OK; } -void COutHandler::InitProps() +void COutHandler::InitProps7z() { - CMultiMethodProps::Init(); - _removeSfxBlock = false; _compressHeaders = true; _encryptHeadersSpecified = false; @@ -723,6 +725,14 @@ void COutHandler::InitProps() _useTypeSorting = false; } +void COutHandler::InitProps() +{ + CMultiMethodProps::Init(); + InitProps7z(); +} + + + HRESULT COutHandler::SetSolidFromString(const UString &s) { UString s2 = s; @@ -763,6 +773,10 @@ HRESULT COutHandler::SetSolidFromString(const UString &s) } _numSolidBytes = (v << numBits); _numSolidBytesDefined = true; + /* + if (_numSolidBytes == 0) + _numSolidFiles = 1; + */ } } return S_OK; @@ -811,7 +825,7 @@ HRESULT COutHandler::SetProperty(const wchar_t *nameSpec, const PROPVARIANT &val return E_INVALIDARG; return SetSolidFromString(name); } - + UInt32 number; int index = ParseStringToUInt32(name, number); // UString realName = name.Ptr(index); @@ -922,3 +936,5 @@ STDMETHODIMP CHandler::SetProperties(const wchar_t * const *names, const PROPVAR } }} + +#endif diff --git a/CPP/7zip/Archive/7z/7zIn.cpp b/CPP/7zip/Archive/7z/7zIn.cpp index 8809236d..bbc77b03 100644 --- a/CPP/7zip/Archive/7z/7zIn.cpp +++ b/CPP/7zip/Archive/7z/7zIn.cpp @@ -1115,11 +1115,11 @@ HRESULT CInArchive::ReadAndDecodePackedStreams( , dataAfterEnd_Error _7Z_DECODER_CRYPRO_VARS - #if !defined(_7ZIP_ST) && !defined(_SFX) + #if !defined(_7ZIP_ST) , false // mtMode , 1 // numThreads + , 0 // memUsage #endif - ); RINOK(result); diff --git a/CPP/7zip/Archive/7z/7zUpdate.cpp b/CPP/7zip/Archive/7z/7zUpdate.cpp index c416b35f..25a9100d 100644 --- a/CPP/7zip/Archive/7z/7zUpdate.cpp +++ b/CPP/7zip/Archive/7z/7zUpdate.cpp @@ -1507,7 +1507,8 @@ void CThreadDecoder::Execute() _7Z_DECODER_CRYPRO_VARS #ifndef _7ZIP_ST - , MtMode, NumThreads + , MtMode, NumThreads, + 0 // MemUsage #endif ); @@ -1696,13 +1697,14 @@ HRESULT Update( UInt64 inSizeForReduce = 0; { + bool isSolid = (numSolidFiles > 1 && options.NumSolidBytes != 0); FOR_VECTOR (i, updateItems) { const CUpdateItem &ui = updateItems[i]; if (ui.NewData) { complexity += ui.Size; - if (numSolidFiles != 1) + if (isSolid) inSizeForReduce += ui.Size; else if (inSizeForReduce < ui.Size) inSizeForReduce = ui.Size; @@ -2142,8 +2144,8 @@ HRESULT Update( #ifndef _7ZIP_ST , false // mtMode , 1 // numThreads + , 0 // memUsage #endif - ); RINOK(res); @@ -2293,7 +2295,8 @@ HRESULT Update( continue; CRecordVector refItems; refItems.ClearAndSetSize(numFiles); - bool sortByType = (options.UseTypeSorting && numSolidFiles > 1); + // bool sortByType = (options.UseTypeSorting && isSoid); // numSolidFiles > 1 + bool sortByType = options.UseTypeSorting; unsigned i; diff --git a/CPP/7zip/Archive/Cab/CabHandler.cpp b/CPP/7zip/Archive/Cab/CabHandler.cpp index 6726783c..d1275e8f 100644 --- a/CPP/7zip/Archive/Cab/CabHandler.cpp +++ b/CPP/7zip/Archive/Cab/CabHandler.cpp @@ -549,9 +549,9 @@ private: Byte *TempBuf; UInt32 TempBufSize; + UInt32 TempBufWritten; unsigned NumIdenticalFiles; bool TempBufMode; - UInt32 m_BufStartFolderOffset; unsigned m_StartIndex; unsigned m_CurrentIndex; @@ -575,7 +575,6 @@ private: HRESULT OpenFile(); HRESULT CloseFileWithResOp(Int32 resOp); HRESULT CloseFile(); - HRESULT Write2(const void *data, UInt32 size, UInt32 *processedSize, bool isOK); public: HRESULT WriteEmptyFiles(); @@ -672,11 +671,11 @@ HRESULT CFolderOutStream::OpenFile() FreeTempBuf(); TempBuf = (Byte *)MyAlloc(item.Size); TempBufSize = item.Size; - if (TempBuf == NULL) + if (!TempBuf) return E_OUTOFMEMORY; } TempBufMode = true; - m_BufStartFolderOffset = item.Offset; + TempBufWritten = 0; } else if (numExtractItems == 1) { @@ -725,8 +724,9 @@ HRESULT CFolderOutStream::WriteEmptyFiles() } -HRESULT CFolderOutStream::Write2(const void *data, UInt32 size, UInt32 *processedSize, bool isOK) +HRESULT CFolderOutStream::Write(const void *data, UInt32 size, UInt32 *processedSize) { + // (data == NULL) means Error_Data for solid folder flushing COM_TRY_BEGIN UInt32 realProcessed = 0; @@ -741,21 +741,34 @@ HRESULT CFolderOutStream::Write2(const void *data, UInt32 size, UInt32 *processe HRESULT res = S_OK; if (numBytesToWrite != 0) { - if (!isOK) + if (!data) m_IsOk = false; + if (m_RealOutStream) { UInt32 processedSizeLocal = 0; - res = m_RealOutStream->Write((const Byte *)data, numBytesToWrite, &processedSizeLocal); + // 18.01 : we don't want ZEROs instead of missing data + if (data) + res = m_RealOutStream->Write((const Byte *)data, numBytesToWrite, &processedSizeLocal); + else + processedSizeLocal = numBytesToWrite; numBytesToWrite = processedSizeLocal; } + if (TempBufMode && TempBuf) - memcpy(TempBuf + (m_PosInFolder - m_BufStartFolderOffset), data, numBytesToWrite); + { + if (data) + { + memcpy(TempBuf + TempBufWritten, data, numBytesToWrite); + TempBufWritten += numBytesToWrite; + } + } } realProcessed += numBytesToWrite; if (processedSize) *processedSize = realProcessed; - data = (const void *)((const Byte *)data + numBytesToWrite); + if (data) + data = (const void *)((const Byte *)data + numBytesToWrite); size -= numBytesToWrite; m_RemainFileSize -= numBytesToWrite; m_PosInFolder += numBytesToWrite; @@ -773,7 +786,7 @@ HRESULT CFolderOutStream::Write2(const void *data, UInt32 size, UInt32 *processe m_FileIsOpen = true; m_CurrentIndex++; if (result == S_OK && m_RealOutStream && TempBuf) - result = WriteStream(m_RealOutStream, TempBuf, (size_t)(m_PosInFolder - m_BufStartFolderOffset)); + result = WriteStream(m_RealOutStream, TempBuf, TempBufWritten); if (!TempBuf && TempBufMode && m_RealOutStream) { @@ -822,7 +835,8 @@ HRESULT CFolderOutStream::Write2(const void *data, UInt32 size, UInt32 *processe realProcessed += numBytesToWrite; if (processedSize) *processedSize = realProcessed; - data = (const void *)((const Byte *)data + numBytesToWrite); + if (data) + data = (const void *)((const Byte *)data + numBytesToWrite); size -= numBytesToWrite; m_PosInFolder += numBytesToWrite; } @@ -843,12 +857,6 @@ HRESULT CFolderOutStream::Write2(const void *data, UInt32 size, UInt32 *processe } -STDMETHODIMP CFolderOutStream::Write(const void *data, UInt32 size, UInt32 *processedSize) -{ - return Write2(data, size, processedSize, true); -} - - HRESULT CFolderOutStream::FlushCorrupted(unsigned folderIndex) { if (!NeedMoreWrite()) @@ -862,19 +870,16 @@ HRESULT CFolderOutStream::FlushCorrupted(unsigned folderIndex) return S_OK; } - const unsigned kBufSize = (1 << 12); - Byte buf[kBufSize]; - for (unsigned i = 0; i < kBufSize; i++) - buf[i] = 0; - for (;;) { if (!NeedMoreWrite()) return S_OK; UInt64 remain = GetRemain(); - UInt32 size = (remain < kBufSize ? (UInt32)remain : (UInt32)kBufSize); + UInt32 size = (UInt32)1 << 20; + if (size > remain) + size = (UInt32)remain; UInt32 processedSizeLocal = 0; - RINOK(Write2(buf, size, &processedSizeLocal, false)); + RINOK(Write(NULL, size, &processedSizeLocal)); } } diff --git a/CPP/7zip/Archive/Common/HandlerOut.cpp b/CPP/7zip/Archive/Common/HandlerOut.cpp index a195be91..41762d90 100644 --- a/CPP/7zip/Archive/Common/HandlerOut.cpp +++ b/CPP/7zip/Archive/Common/HandlerOut.cpp @@ -2,18 +2,92 @@ #include "StdAfx.h" -#ifndef _7ZIP_ST -#include "../../../Windows/System.h" -#endif +#include "../../../Common/StringToInt.h" #include "../Common/ParseProperties.h" #include "HandlerOut.h" -using namespace NWindows; - namespace NArchive { +bool ParseSizeString(const wchar_t *s, const PROPVARIANT &prop, UInt64 percentsBase, UInt64 &res) +{ + if (*s == 0) + { + switch (prop.vt) + { + case VT_UI4: res = prop.ulVal; return true; + case VT_UI8: res = prop.uhVal.QuadPart; return true; + case VT_BSTR: + s = prop.bstrVal; + break; + default: return false; + } + } + else if (prop.vt != VT_EMPTY) + return false; + + const wchar_t *end; + UInt64 v = ConvertStringToUInt64(s, &end); + if (s == end) + return false; + wchar_t c = *end; + if (c == 0) + { + res = v; + return true; + } + if (end[1] != 0) + return false; + + if (c == '%') + { + res = percentsBase / 100 * v; + return true; + } + + unsigned numBits; + switch (MyCharLower_Ascii(c)) + { + case 'b': numBits = 0; break; + case 'k': numBits = 10; break; + case 'm': numBits = 20; break; + case 'g': numBits = 30; break; + case 't': numBits = 40; break; + default: return false; + } + UInt64 val2 = v << numBits; + if ((val2 >> numBits) != v) + return false; + res = val2; + return true; +} + +bool CCommonMethodProps::SetCommonProperty(const UString &name, const PROPVARIANT &value, HRESULT &hres) +{ + hres = S_OK; + + if (name.IsPrefixedBy_Ascii_NoCase("mt")) + { + #ifndef _7ZIP_ST + hres = ParseMtProp(name.Ptr(2), value, _numProcessors, _numThreads); + #endif + return true; + } + + if (name.IsPrefixedBy_Ascii_NoCase("memuse")) + { + if (!ParseSizeString(name.Ptr(6), value, _memAvail, _memUsage)) + hres = E_INVALIDARG; + return true; + } + + return false; +} + + +#ifndef EXTRACT_ONLY + static void SetMethodProp32(COneMethodInfo &m, PROPID propID, UInt32 value) { if (m.FindProp(propID) < 0) @@ -34,21 +108,23 @@ void CMultiMethodProps::SetMethodThreadsTo(COneMethodInfo &oneMethodInfo, UInt32 } #endif -void CMultiMethodProps::Init() +void CMultiMethodProps::InitMulti() { - #ifndef _7ZIP_ST - _numProcessors = _numThreads = NSystem::GetNumberOfProcessors(); - #endif - _level = (UInt32)(Int32)-1; _analysisLevel = -1; - - _autoFilter = true; _crcSize = 4; - _filterMethod.Clear(); - _methods.Clear(); + _autoFilter = true; } +void CMultiMethodProps::Init() +{ + InitCommon(); + InitMulti(); + _methods.Clear(); + _filterMethod.Clear(); +} + + HRESULT CMultiMethodProps::SetProperty(const wchar_t *nameSpec, const PROPVARIANT &value) { UString name = nameSpec; @@ -78,20 +154,18 @@ HRESULT CMultiMethodProps::SetProperty(const wchar_t *nameSpec, const PROPVARIAN _crcSize = 4; return ParsePropToUInt32(name, value, _crcSize); } + + { + HRESULT hres; + if (SetCommonProperty(name, value, hres)) + return hres; + } UInt32 number; unsigned index = ParseStringToUInt32(name, number); UString realName = name.Ptr(index); if (index == 0) { - if (name.IsPrefixedBy_Ascii_NoCase("mt")) - { - #ifndef _7ZIP_ST - RINOK(ParseMtProp(name.Ptr(2), value, _numProcessors, _numThreads)); - #endif - - return S_OK; - } if (name.IsEqualTo("f")) { HRESULT res = PROPVARIANT_to_bool(value, _autoFilter); @@ -110,20 +184,20 @@ HRESULT CMultiMethodProps::SetProperty(const wchar_t *nameSpec, const PROPVARIAN return _methods[number].ParseMethodFromPROPVARIANT(realName, value); } + + void CSingleMethodProps::Init() { + InitCommon(); + InitSingle(); Clear(); - _level = (UInt32)(Int32)-1; - - #ifndef _7ZIP_ST - _numProcessors = _numThreads = NWindows::NSystem::GetNumberOfProcessors(); - AddProp_NumThreads(_numThreads); - #endif } + HRESULT CSingleMethodProps::SetProperties(const wchar_t * const *names, const PROPVARIANT *values, UInt32 numProps) { Init(); + for (UInt32 i = 0; i < numProps; i++) { UString name = names[i]; @@ -137,20 +211,22 @@ HRESULT CSingleMethodProps::SetProperties(const wchar_t * const *names, const PR RINOK(ParsePropToUInt32(name.Ptr(1), value, a)); _level = a; AddProp_Level(a); + continue; } - else if (name.IsPrefixedBy_Ascii_NoCase("mt")) { - #ifndef _7ZIP_ST - RINOK(ParseMtProp(name.Ptr(2), value, _numProcessors, _numThreads)); - AddProp_NumThreads(_numThreads); - #endif - } - else - { - RINOK(ParseMethodFromPROPVARIANT(names[i], value)); + HRESULT hres; + if (SetCommonProperty(name, value, hres)) + { + RINOK(hres) + continue; + } } + RINOK(ParseMethodFromPROPVARIANT(names[i], value)); } + return S_OK; } +#endif + } diff --git a/CPP/7zip/Archive/Common/HandlerOut.h b/CPP/7zip/Archive/Common/HandlerOut.h index da81ed60..90b000ac 100644 --- a/CPP/7zip/Archive/Common/HandlerOut.h +++ b/CPP/7zip/Archive/Common/HandlerOut.h @@ -3,20 +3,57 @@ #ifndef __HANDLER_OUT_H #define __HANDLER_OUT_H +#include "../../../Windows/System.h" + #include "../../Common/MethodProps.h" namespace NArchive { -class CMultiMethodProps +bool ParseSizeString(const wchar_t *name, const PROPVARIANT &prop, UInt64 percentsBase, UInt64 &res); + +class CCommonMethodProps { - UInt32 _level; - int _analysisLevel; +protected: + void InitCommon() + { + #ifndef _7ZIP_ST + _numProcessors = _numThreads = NWindows::NSystem::GetNumberOfProcessors(); + #endif + + UInt64 memAvail = (UInt64)(sizeof(size_t)) << 28; + _memAvail = memAvail; + _memUsage = memAvail; + if (NWindows::NSystem::GetRamSize(memAvail)) + { + _memAvail = memAvail; + _memUsage = memAvail / 32 * 17; + } + } + public: #ifndef _7ZIP_ST UInt32 _numThreads; UInt32 _numProcessors; #endif + UInt64 _memUsage; + UInt64 _memAvail; + + bool SetCommonProperty(const UString &name, const PROPVARIANT &value, HRESULT &hres); + + CCommonMethodProps() { InitCommon(); } +}; + + +#ifndef EXTRACT_ONLY + +class CMultiMethodProps: public CCommonMethodProps +{ + UInt32 _level; + int _analysisLevel; + + void InitMulti(); +public: UInt32 _crcSize; CObjectVector _methods; COneMethodInfo _filterMethod; @@ -43,27 +80,31 @@ public: int GetAnalysisLevel() const { return _analysisLevel; } void Init(); + CMultiMethodProps() { InitMulti(); } - CMultiMethodProps() { Init(); } HRESULT SetProperty(const wchar_t *name, const PROPVARIANT &value); }; -class CSingleMethodProps: public COneMethodInfo + +class CSingleMethodProps: public COneMethodInfo, public CCommonMethodProps { UInt32 _level; - -public: - #ifndef _7ZIP_ST - UInt32 _numThreads; - UInt32 _numProcessors; - #endif + void InitSingle() + { + _level = (UInt32)(Int32)-1; + } + +public: void Init(); - CSingleMethodProps() { Init(); } + CSingleMethodProps() { InitSingle(); } + int GetLevel() const { return _level == (UInt32)(Int32)-1 ? 5 : (int)_level; } HRESULT SetProperties(const wchar_t * const *names, const PROPVARIANT *values, UInt32 numProps); }; +#endif + } #endif diff --git a/CPP/7zip/Archive/DmgHandler.cpp b/CPP/7zip/Archive/DmgHandler.cpp index 8486804a..5d7415ee 100644 --- a/CPP/7zip/Archive/DmgHandler.cpp +++ b/CPP/7zip/Archive/DmgHandler.cpp @@ -19,6 +19,7 @@ #include "../Compress/BZip2Decoder.h" #include "../Compress/CopyCoder.h" +#include "../Compress/LzfseDecoder.h" #include "../Compress/ZlibDecoder.h" #include "Common/OutStreamWithCRC.h" @@ -121,6 +122,7 @@ enum METHOD_ADC = 0x80000004, METHOD_ZLIB = 0x80000005, METHOD_BZIP2 = 0x80000006, + METHOD_LZFSE = 0x80000007, METHOD_COMMENT = 0x7FFFFFFE, // is used to comment "+beg" and "+end" in extra field. METHOD_END = 0xFFFFFFFF }; @@ -217,6 +219,7 @@ class CHandler: bool _masterCrcError; bool _headersError; + UInt32 _dataStartOffset; UInt64 _startPos; UInt64 _phySize; @@ -276,6 +279,7 @@ void CMethods::GetString(AString &res) const case METHOD_ADC: s = "ADC"; break; case METHOD_ZLIB: s = "ZLIB"; break; case METHOD_BZIP2: s = "BZip2"; break; + case METHOD_LZFSE: s = "LZFSE"; break; default: ConvertUInt32ToString(type, buf); s = buf; } res.Add_OptSpaced(s); @@ -307,6 +311,10 @@ static const CAppleName k_Names[] = { true, "hfs", "Apple_HFS" }, { true, "hfsx", "Apple_HFSX" }, { true, "ufs", "Apple_UFS" }, + + // efi_sys partition is FAT32, but it's not main file. So we use (IsFs = false) + { false, "efi_sys", "C12A7328-F81F-11D2-BA4B-00A0C93EC93B" }, + { false, "free", "Apple_Free" }, { false, "ddm", "DDM" }, { false, NULL, "Apple_partition_map" }, @@ -326,6 +334,7 @@ static const Byte kProps[] = kpidCRC, kpidComment, kpidMethod + // kpidOffset }; IMP_IInArchive_Props @@ -624,17 +633,40 @@ bool CHandler::ParseBlob(const CByteBuffer &data) HRESULT CHandler::Open2(IInStream *stream) { + /* + - usual dmg contains Koly Header at the end: + - rare case dmg contains Koly Header at the start. + */ + + _dataStartOffset = 0; RINOK(stream->Seek(0, STREAM_SEEK_CUR, &_startPos)); + UInt64 fileSize = 0; + RINOK(stream->Seek(0, STREAM_SEEK_END, &fileSize)); + RINOK(stream->Seek(_startPos, STREAM_SEEK_SET, NULL)); + Byte buf[HEADER_SIZE]; RINOK(ReadStream_FALSE(stream, buf, HEADER_SIZE)); UInt64 headerPos; + bool startKolyMode = false; + if (IsKoly(buf)) + { + // it can be normal koly-at-the-end or koly-at-the-start headerPos = _startPos; + if (_startPos <= (1 << 8)) + { + // we want to support startKolyMode, even if there is + // some data before dmg file, like 128 bytes MacBin header + _dataStartOffset = HEADER_SIZE; + startKolyMode = true; + } + } else { - RINOK(stream->Seek(0, STREAM_SEEK_END, &headerPos)); + // we check only koly-at-the-end + headerPos = fileSize; if (headerPos < HEADER_SIZE) return S_FALSE; headerPos -= HEADER_SIZE; @@ -660,24 +692,35 @@ HRESULT CHandler::Open2(IInStream *stream) // CChecksum dataForkChecksum; // dataForkChecksum.Parse(buf + 0x50); - _startPos = 0; - UInt64 top = 0; - if (!dataForkPair.UpdateTop(headerPos, top)) return S_FALSE; - if (!xmlPair.UpdateTop(headerPos, top)) return S_FALSE; - if (!rsrcPair.UpdateTop(headerPos, top)) return S_FALSE; + UInt64 limit = startKolyMode ? fileSize : headerPos; + + if (!dataForkPair.UpdateTop(limit, top)) return S_FALSE; + if (!xmlPair.UpdateTop(limit, top)) return S_FALSE; + if (!rsrcPair.UpdateTop(limit, top)) return S_FALSE; /* Some old dmg files contain garbage data in blobPair field. So we need to ignore such garbage case; And we still need to detect offset of start of archive for "parser" mode. */ - bool useBlob = blobPair.UpdateTop(headerPos, top); + bool useBlob = blobPair.UpdateTop(limit, top); - _startPos = 0; - _phySize = headerPos + HEADER_SIZE; - if (top != headerPos) + if (startKolyMode) + _phySize = top; + else { + _phySize = headerPos + HEADER_SIZE; + _startPos = 0; + if (top != headerPos) + { + /* + if expected absolute offset is not equal to real header offset, + 2 cases are possible: + - additional (unknown) headers + - archive with offset. + So we try to read XML with absolute offset to select from these two ways. + */ CForkPair xmlPair2 = xmlPair; const char *sz = " adcCoder = adcCoderSpec; + NCompress::NLzfse::CDecoder *lzfseCoderSpec = new NCompress::NLzfse::CDecoder(); + CMyComPtr lzfseCoder = lzfseCoderSpec; + CLocalProgress *lps = new CLocalProgress; CMyComPtr progress = lps; lps->Init(extractCallback, false); @@ -1369,7 +1430,7 @@ STDMETHODIMP CHandler::Extract(const UInt32 *indices, UInt32 numItems, break; } - RINOK(_inStream->Seek(_startPos + item.StartPos + block.PackPos, STREAM_SEEK_SET, NULL)); + RINOK(_inStream->Seek(_startPos + _dataStartOffset + item.StartPos + block.PackPos, STREAM_SEEK_SET, NULL)); streamSpec->Init(block.PackSize); bool realMethod = true; outStreamSpec->Init(block.UnpSize); @@ -1419,6 +1480,12 @@ STDMETHODIMP CHandler::Extract(const UInt32 *indices, UInt32 numItems, opRes = NExtract::NOperationResult::kDataError; break; } + + case METHOD_LZFSE: + { + res = lzfseCoder->Code(inStream, outStream, &block.PackSize, &block.UnpSize, progress); + break; + } default: opRes = NExtract::NOperationResult::kUnsupportedMethod; @@ -1490,6 +1557,9 @@ class CInStream: CAdcDecoder *adcCoderSpec; CMyComPtr adcCoder; + NCompress::NLzfse::CDecoder *lzfseCoderSpec; + CMyComPtr lzfseCoder; + CBufPtrSeqOutStream *outStreamSpec; CMyComPtr outStream; @@ -1651,6 +1721,15 @@ STDMETHODIMP CInStream::Read(void *data, UInt32 size, UInt32 *processedSize) if (res == S_OK && bzip2CoderSpec->GetInputProcessedSize() != block.PackSize) res = S_FALSE; break; + + case METHOD_LZFSE: + if (!lzfseCoder) + { + lzfseCoderSpec = new NCompress::NLzfse::CDecoder(); + lzfseCoder = lzfseCoderSpec; + } + res = lzfseCoder->Code(inStream, outStream, &block.PackSize, &block.UnpSize, NULL); + break; default: return E_FAIL; @@ -1738,6 +1817,7 @@ STDMETHODIMP CHandler::GetStream(UInt32 index, ISequentialInStream **stream) case METHOD_ADC: case METHOD_ZLIB: case METHOD_BZIP2: + case METHOD_LZFSE: case METHOD_END: break; default: @@ -1747,7 +1827,7 @@ STDMETHODIMP CHandler::GetStream(UInt32 index, ISequentialInStream **stream) spec->Stream = _inStream; spec->Size = spec->File->Size; - RINOK(spec->InitAndSeek(_startPos)); + RINOK(spec->InitAndSeek(_startPos + _dataStartOffset)); *stream = specStream.Detach(); return S_OK; diff --git a/CPP/7zip/Archive/ElfHandler.cpp b/CPP/7zip/Archive/ElfHandler.cpp index a3b7f3f4..08771e83 100644 --- a/CPP/7zip/Archive/ElfHandler.cpp +++ b/CPP/7zip/Archive/ElfHandler.cpp @@ -567,6 +567,7 @@ static const char * const g_Machines[] = static const CUInt32PCharPair g_MachinePairs[] = { + { 243, "RISC-V" }, { 47787, "Xilinx MicroBlaze" } // { 0x9026, "Alpha" } }; diff --git a/CPP/7zip/Archive/HfsHandler.cpp b/CPP/7zip/Archive/HfsHandler.cpp index 02cfb0d7..83a0d413 100644 --- a/CPP/7zip/Archive/HfsHandler.cpp +++ b/CPP/7zip/Archive/HfsHandler.cpp @@ -317,11 +317,15 @@ public: // bool CaseSensetive; UString ResFileName; + UInt64 SpecOffset; UInt64 PhySize; + UInt64 PhySize2; void Clear() { + SpecOffset = 0; PhySize = 0; + PhySize2 = 0; HeadersError = false; ThereAreAltStreams = false; // CaseSensetive = false; @@ -444,7 +448,7 @@ HRESULT CDatabase::ReadFile(const CFork &fork, CByteBuffer &buf, IInStream *inSt e.NumBlocks > fork.NumBlocks - curBlock || e.NumBlocks > Header.NumBlocks - e.Pos) return S_FALSE; - RINOK(inStream->Seek((UInt64)e.Pos << Header.BlockSizeLog, STREAM_SEEK_SET, NULL)); + RINOK(inStream->Seek(SpecOffset + ((UInt64)e.Pos << Header.BlockSizeLog), STREAM_SEEK_SET, NULL)); RINOK(ReadStream_FALSE(inStream, (Byte *)buf + ((size_t)curBlock << Header.BlockSizeLog), (size_t)e.NumBlocks << Header.BlockSizeLog)); @@ -1154,13 +1158,36 @@ HRESULT CDatabase::LoadCatalog(const CFork &fork, const CObjectVector 5) + return k_IsArc_Res_NO; + } + return k_IsArc_Res_YES; +} +} HRESULT CDatabase::Open2(IInStream *inStream, IArchiveOpenCallback *progress) { Clear(); - static const unsigned kHeaderSize = kHeaderPadSize + 512; - Byte buf[kHeaderSize]; - RINOK(ReadStream_FALSE(inStream, buf, kHeaderSize)); + Byte buf[kHfsHeaderSize]; + RINOK(ReadStream_FALSE(inStream, buf, kHfsHeaderSize)); { for (unsigned i = 0; i < kHeaderPadSize; i++) if (buf[i] != 0) @@ -1171,6 +1198,67 @@ HRESULT CDatabase::Open2(IInStream *inStream, IArchiveOpenCallback *progress) h.Header[0] = p[0]; h.Header[1] = p[1]; + + if (p[0] == 'B' && p[1] == 'D') + { + /* + It's header for old HFS format. + We don't support old HFS format, but we support + special HFS volume that contains embedded HFS+ volume + */ + + if (p[0x7C] != 'H' || p[0x7C + 1] != '+') + return S_FALSE; + + /* + h.CTime = Get32(p + 0x2); + h.MTime = Get32(p + 0x6); + + h.NumFiles = Get32(p + 0x54); + h.NumFolders = Get32(p + 0x58); + + if (h.NumFolders > ((UInt32)1 << 29) || + h.NumFiles > ((UInt32)1 << 30)) + return S_FALSE; + if (progress) + { + UInt64 numFiles = (UInt64)h.NumFiles + h.NumFolders + 1; + RINOK(progress->SetTotal(&numFiles, NULL)); + } + h.NumFreeBlocks = Get16(p + 0x22); + */ + + UInt32 blockSize = Get32(p + 0x14); + + { + unsigned i; + for (i = 9; ((UInt32)1 << i) != blockSize; i++) + if (i == 31) + return S_FALSE; + h.BlockSizeLog = i; + } + + h.NumBlocks = Get16(p + 0x12); + /* + we suppose that it has the follwing layout + { + start block with header + [h.NumBlocks] + end block with header + } + */ + PhySize2 = ((UInt64)h.NumBlocks + 2) << h.BlockSizeLog; + + UInt32 startBlock = Get16(p + 0x7C + 2); + UInt32 blockCount = Get16(p + 0x7C + 4); + SpecOffset = (UInt64)(1 + startBlock) << h.BlockSizeLog; + UInt64 phy = SpecOffset + ((UInt64)blockCount << h.BlockSizeLog); + if (PhySize2 < phy) + PhySize2 = phy; + RINOK(inStream->Seek(SpecOffset, STREAM_SEEK_SET, NULL)); + RINOK(ReadStream_FALSE(inStream, buf, kHfsHeaderSize)); + } + if (p[0] != 'H' || (p[1] != '+' && p[1] != 'X')) return S_FALSE; h.Version = Get16(p + 2); @@ -1330,7 +1418,14 @@ STDMETHODIMP CHandler::GetArchiveProperty(PROPID propID, PROPVARIANT *value) { case kpidExtension: prop = Header.IsHfsX() ? "hfsx" : "hfs"; break; case kpidMethod: prop = Header.IsHfsX() ? "HFSX" : "HFS+"; break; - case kpidPhySize: prop = PhySize; break; + case kpidPhySize: + { + UInt64 v = SpecOffset + PhySize; + if (v < PhySize2) + v = PhySize2; + prop = v; + break; + } case kpidClusterSize: prop = (UInt32)1 << Header.BlockSizeLog; break; case kpidFreeSpace: prop = (UInt64)Header.GetFreeSize(); break; case kpidMTime: HfsTimeToProp(Header.MTime, prop); break; @@ -1754,7 +1849,7 @@ STDMETHODIMP CHandler::Extract(const UInt32 *indices, UInt32 numItems, if (fork.Size == pos) break; const CExtent &e = fork.Extents[extentIndex]; - RINOK(_stream->Seek((UInt64)e.Pos << Header.BlockSizeLog, STREAM_SEEK_SET, NULL)); + RINOK(_stream->Seek(SpecOffset + ((UInt64)e.Pos << Header.BlockSizeLog), STREAM_SEEK_SET, NULL)); UInt64 extentRem = (UInt64)e.NumBlocks << Header.BlockSizeLog; while (extentRem != 0) { @@ -1865,6 +1960,7 @@ STDMETHODIMP CHandler::GetStream(UInt32 index, ISequentialInStream **stream) } static const Byte k_Signature[] = { + 2, 'B', 'D', 4, 'H', '+', 0, 4, 4, 'H', 'X', 0, 5 }; @@ -1873,6 +1969,6 @@ REGISTER_ARC_I( k_Signature, kHeaderPadSize, NArcInfoFlags::kMultiSignature, - NULL) + IsArc_HFS) }} diff --git a/CPP/7zip/Archive/IArchive.h b/CPP/7zip/Archive/IArchive.h index ed035374..0c35349a 100644 --- a/CPP/7zip/Archive/IArchive.h +++ b/CPP/7zip/Archive/IArchive.h @@ -488,6 +488,16 @@ ARCHIVE_INTERFACE(IOutArchive, 0xA0) }; +/* +ISetProperties::SetProperties() + PROPVARIANT values[i].vt: + VT_EMPTY + VT_BOOL + VT_UI4 - if 32-bit number + VT_UI8 - if 64-bit number + VT_BSTR +*/ + ARCHIVE_INTERFACE(ISetProperties, 0x03) { STDMETHOD(SetProperties)(const wchar_t * const *names, const PROPVARIANT *values, UInt32 numProps) PURE; diff --git a/CPP/7zip/Archive/Iso/IsoIn.cpp b/CPP/7zip/Archive/Iso/IsoIn.cpp index 9dd4c3f1..699c7a1a 100644 --- a/CPP/7zip/Archive/Iso/IsoIn.cpp +++ b/CPP/7zip/Archive/Iso/IsoIn.cpp @@ -180,7 +180,7 @@ UInt32 CInArchive::ReadDigits(int numDigits) Byte b = ReadByte(); if (b < '0' || b > '9') { - if (b == 0) // it's bug in some CD's + if (b == 0 || b == ' ') // it's bug in some CD's b = '0'; else throw CHeaderErrorException(); diff --git a/CPP/7zip/Archive/PeHandler.cpp b/CPP/7zip/Archive/PeHandler.cpp index b8c8fa65..2457009a 100644 --- a/CPP/7zip/Archive/PeHandler.cpp +++ b/CPP/7zip/Archive/PeHandler.cpp @@ -109,7 +109,8 @@ void CVersion::ToProp(NCOM::CPropVariant &prop) prop = sz; } -static const unsigned kHeaderSize = 4 + 20; +static const unsigned kCoffHeaderSize = 20; +static const unsigned kPeHeaderSize = 4 + kCoffHeaderSize; static const unsigned k_OptHeader32_Size_MIN = 96; static const unsigned k_OptHeader64_Size_MIN = 112; @@ -125,15 +126,14 @@ struct CHeader UInt16 OptHeaderSize; UInt16 Flags; - bool Parse(const Byte *p); + void ParseBase(const Byte *p); + bool ParseCoff(const Byte *p); + bool ParsePe(const Byte *p); bool IsDll() const { return (Flags & PE_IMAGE_FILE_DLL) != 0; } }; -bool CHeader::Parse(const Byte *p) +void CHeader::ParseBase(const Byte *p) { - if (Get32(p) != k_Signature32) - return false; - p += 4; G16( 0, Machine); G16( 2, NumSections); G32( 4, Time); @@ -141,6 +141,13 @@ bool CHeader::Parse(const Byte *p) G32(12, NumSymbols); G16(16, OptHeaderSize); G16(18, Flags); +} + +bool CHeader::ParsePe(const Byte *p) +{ + if (Get32(p) != k_Signature32) + return false; + ParseBase(p + 4); return OptHeaderSize >= k_OptHeader32_Size_MIN; } @@ -380,6 +387,10 @@ void CSection::Parse(const Byte *p) G32(36, Flags); } + + +// IMAGE_FILE_* + static const CUInt32PCharPair g_HeaderCharacts[] = { { 1, "Executable" }, @@ -399,39 +410,65 @@ static const CUInt32PCharPair g_HeaderCharacts[] = { 15, "Big-Endian" } }; -static const CUInt32PCharPair g_DllCharacts[] = +// IMAGE_DLLCHARACTERISTICS_* + +static const char * const g_DllCharacts[] = { - { 5, "HighEntropyVA" }, - { 6, "Relocated" }, - { 7, "Integrity" }, - { 8, "NX-Compatible" }, - { 9, "NoIsolation" }, - { 10, "NoSEH" }, - { 11, "NoBind" }, - { 12, "AppContainer" }, - { 13, "WDM" }, - { 14, "GuardCF" }, - { 15, "TerminalServerAware" } + NULL + , NULL + , NULL + , NULL + , NULL + , "HighEntropyVA" + , "Relocated" + , "Integrity" + , "NX-Compatible" + , "NoIsolation" + , "NoSEH" + , "NoBind" + , "AppContainer" + , "WDM" + , "GuardCF" + , "TerminalServerAware" }; -static const CUInt32PCharPair g_SectFlags[] = + +// IMAGE_SCN_* constants: + +static const char * const g_SectFlags[] = { - { 3, "NoPad" }, - { 5, "Code" }, - { 6, "InitializedData" }, - { 7, "UninitializedData" }, - { 9, "Comments" }, - { 11, "Remove" }, - { 12, "COMDAT" }, - { 15, "GP" }, - { 24, "ExtendedRelocations" }, - { 25, "Discardable" }, - { 26, "NotCached" }, - { 27, "NotPaged" }, - { 28, "Shared" }, - { 29, "Execute" }, - { 30, "Read" }, - { 31, "Write" } + NULL + , NULL + , NULL + , "NoPad" + , NULL + , "Code" + , "InitializedData" + , "UninitializedData" + , "Other" + , "Comments" + , NULL // OVER + , "Remove" + , "COMDAT" + , NULL + , "NO_DEFER_SPEC_EXC" + , "GP" // MEM_FARDATA + , NULL // SYSHEAP + , "PURGEABLE" // 16BIT + , "LOCKED" + , "PRELOAD" + , NULL + , NULL + , NULL + , NULL + , "ExtendedRelocations" + , "Discardable" + , "NotCached" + , "NotPaged" + , "Shared" + , "Execute" + , "Read" + , "Write" }; static const CUInt32PCharPair g_MachinePairs[] = @@ -699,7 +736,6 @@ class CHandler: { CMyComPtr _stream; CObjectVector _sections; - UInt32 _peOffset; CHeader _header; UInt32 _totalSize; Int32 _mainSubfile; @@ -720,9 +756,12 @@ class CHandler: bool _parseResources; bool _checksumError; + bool IsOpt() const { return _header.OptHeaderSize != 0; } + COptHeader _optHeader; bool _allowTail; + bool _coffMode; HRESULT LoadDebugSections(IInStream *stream, bool &thereIsSection); HRESULT Open2(IInStream *stream, IArchiveOpenCallback *callback); @@ -742,7 +781,10 @@ class CHandler: } public: - CHandler(): _allowTail(false) {} + CHandler(bool coffMode = false): + _coffMode(coffMode), + _allowTail(coffMode) + {} MY_UNKNOWN_IMP3(IInArchive, IInArchiveGetStream, IArchiveAllowTail) INTERFACE_IInArchive(;) @@ -841,6 +883,34 @@ STDMETHODIMP CHandler::GetArchiveProperty(PROPID propID, PROPVARIANT *value) NCOM::CPropVariant prop; switch (propID) { + case kpidPhySize: prop = _totalSize; break; + case kpidComment: if (!_versionFullString.IsEmpty()) prop = _versionFullString; break; + case kpidShortComment: + if (!_versionShortString.IsEmpty()) + prop = _versionShortString; + else + { + PAIR_TO_PROP(g_MachinePairs, _header.Machine, prop); + } + break; + + case kpidName: if (!_originalFilename.IsEmpty()) prop = _originalFilename; break; + + // case kpidIsSelfExe: prop = !_header.IsDll(); break; + // case kpidError: + case kpidWarning: if (_checksumError) prop = "Checksum error"; break; + + case kpidCpu: PAIR_TO_PROP(g_MachinePairs, _header.Machine, prop); break; + case kpidMTime: + case kpidCTime: TimeToProp(_header.Time, prop); break; + case kpidCharacts: FLAGS_TO_PROP(g_HeaderCharacts, _header.Flags, prop); break; + case kpidMainSubfile: if (_mainSubfile >= 0) prop = (UInt32)_mainSubfile; break; + + default: + if (IsOpt()) + switch (propID) + { + case kpidSectAlign: prop = _optHeader.SectAlign; break; case kpidFileAlign: prop = _optHeader.FileAlign; break; case kpidLinkerVer: @@ -857,37 +927,19 @@ STDMETHODIMP CHandler::GetArchiveProperty(PROPID propID, PROPVARIANT *value) case kpidInitDataSize: prop = _optHeader.InitDataSize; break; case kpidUnInitDataSize: prop = _optHeader.UninitDataSize; break; case kpidImageSize: prop = _optHeader.ImageSize; break; - case kpidPhySize: prop = _totalSize; break; case kpidHeadersSize: prop = _optHeader.HeadersSize; break; case kpidChecksum: prop = _optHeader.CheckSum; break; - case kpidComment: if (!_versionFullString.IsEmpty()) prop = _versionFullString; break; - case kpidShortComment: - if (!_versionShortString.IsEmpty()) - prop = _versionShortString; - else - { - PAIR_TO_PROP(g_MachinePairs, _header.Machine, prop); - } - break; - case kpidName: if (!_originalFilename.IsEmpty()) prop = _originalFilename; break; case kpidExtension: if (_header.IsDll()) - prop = _optHeader.IsSybSystem_EFI() ? "efi" : "dll"; + prop = "dll"; + else if (_optHeader.IsSybSystem_EFI()) + prop = "efi"; break; - - // case kpidIsSelfExe: prop = !_header.IsDll(); break; - // case kpidError: - case kpidWarning: if (_checksumError) prop = "Checksum error"; break; - - case kpidCpu: PAIR_TO_PROP(g_MachinePairs, _header.Machine, prop); break; case kpidBit64: if (_optHeader.Is64Bit()) prop = true; break; case kpidSubSystem: TYPE_TO_PROP(g_SubSystems, _optHeader.SubSystem, prop); break; - case kpidMTime: - case kpidCTime: TimeToProp(_header.Time, prop); break; - case kpidCharacts: FLAGS_TO_PROP(g_HeaderCharacts, _header.Flags, prop); break; case kpidDllCharacts: FLAGS_TO_PROP(g_DllCharacts, _optHeader.DllCharacts, prop); break; case kpidStackReserve: prop = _optHeader.StackReserve; break; case kpidStackCommit: prop = _optHeader.StackCommit; break; @@ -898,8 +950,7 @@ STDMETHODIMP CHandler::GetArchiveProperty(PROPID propID, PROPVARIANT *value) // case kpidAddressOfEntryPoint: prop = _optHeader.AddressOfEntryPoint; break; // case kpidBaseOfCode: prop = _optHeader.BaseOfCode; break; // case kpidBaseOfData32: if (!_optHeader.Is64Bit()) prop = _optHeader.BaseOfData32; break; - - case kpidMainSubfile: if (_mainSubfile >= 0) prop = (UInt32)_mainSubfile; break; + } } prop.Detach(value); return S_OK; @@ -1056,7 +1107,24 @@ STDMETHODIMP CHandler::GetProperty(UInt32 index, PROPID propID, PROPVARIANT *val case kpidMTime: case kpidCTime: TimeToProp(item.IsDebug ? item.Time : _header.Time, prop); break; - case kpidCharacts: if (item.IsRealSect) FLAGS_TO_PROP(g_SectFlags, item.Flags, prop); break; + case kpidCharacts: + if (item.IsRealSect) + { + UInt32 flags = item.Flags; + const UInt32 MY__IMAGE_SCN_ALIGN_MASK = 0x00F00000; + AString s = FlagsToString(g_SectFlags, ARRAY_SIZE(g_SectFlags), item.Flags & ~MY__IMAGE_SCN_ALIGN_MASK); + const UInt32 align = ((flags >> 20) & 0xF); + if (align != 0) + { + char sz[32]; + ConvertUInt32ToString(1 << (align - 1), sz); + s.Add_Space(); + s += "align_"; + s += sz; + } + prop = s; + } + break; case kpidZerosTailIsAllowed: if (!item.IsRealSect) prop = true; break; } } @@ -1073,8 +1141,17 @@ HRESULT CHandler::LoadDebugSections(IInStream *stream, bool &thereIsSection) return S_OK; const unsigned kEntrySize = 28; UInt32 numItems = debugLink.Size / kEntrySize; - if (numItems * kEntrySize != debugLink.Size || numItems > 16) + if (numItems > 16) return S_FALSE; + + // MAC's EFI file: numItems can be incorrect. Only first CDebugEntry entry is correct. + // debugLink.Size = kEntrySize + some_data, pointed by entry[0]. + if (numItems * kEntrySize != debugLink.Size) + { + // return S_FALSE; + if (numItems > 1) + numItems = 1; + } UInt64 pa = 0; unsigned i; @@ -2114,6 +2191,31 @@ HRESULT CHandler::OpenResources(unsigned sectionIndex, IInStream *stream, IArchi return S_OK; } + +bool CHeader::ParseCoff(const Byte *p) +{ + ParseBase(p); + if (PointerToSymbolTable < kCoffHeaderSize) + return false; + if (NumSymbols >= (1 << 24)) + return false; + if (OptHeaderSize != 0 && OptHeaderSize < k_OptHeader32_Size_MIN) + return false; + + // 18.04: we reduce false detections + if (NumSections == 0 && OptHeaderSize == 0) + return false; + + for (unsigned i = 0; i < ARRAY_SIZE(g_MachinePairs); i++) + if (Machine == g_MachinePairs[i].Value) + return true; + if (Machine == 0) + return true; + + return false; +} + + static inline bool CheckPeOffset(UInt32 pe) { // ((pe & 7) == 0) is for most PE files. But there is unusual EFI-PE file that uses unaligned pe value. @@ -2133,10 +2235,10 @@ API_FUNC_static_IsArc IsArc_Pe(const Byte *p, size_t size) UInt32 pe = Get32(p + 0x3C); if (!CheckPeOffset(pe)) return k_IsArc_Res_NO; - if (pe + kHeaderSize > size) + if (pe + kPeHeaderSize > size) return k_IsArc_Res_NEED_MORE; CHeader header; - if (!header.Parse(p + pe)) + if (!header.ParsePe(p + pe)) return k_IsArc_Res_NO; return k_IsArc_Res_YES; } @@ -2144,32 +2246,47 @@ API_FUNC_static_IsArc IsArc_Pe(const Byte *p, size_t size) HRESULT CHandler::Open2(IInStream *stream, IArchiveOpenCallback *callback) { + UInt32 coffOffset = 0; + if (_coffMode) { - Byte h[kStartSize]; - _mainSubfile = -1; - RINOK(ReadStream_FALSE(stream, h, kStartSize)); - if (h[0] != 'M' || h[1] != 'Z') - return S_FALSE; - /* most of PE files contain 0x0090 at offset 2. - But some rare PE files contain another values. So we don't use that check. - if (Get16(h + 2) != 0x90) return false; */ - _peOffset = Get32(h + 0x3C); - if (!CheckPeOffset(_peOffset)) + Byte h[kCoffHeaderSize]; + RINOK(ReadStream_FALSE(stream, h, kCoffHeaderSize)); + if (!_header.ParseCoff(h)) return S_FALSE; } + else { - Byte h[kHeaderSize]; - RINOK(stream->Seek(_peOffset, STREAM_SEEK_SET, NULL)); - RINOK(ReadStream_FALSE(stream, h, kHeaderSize)); - if (!_header.Parse(h)) - return S_FALSE; + UInt32 _peOffset; + { + Byte h[kStartSize]; + RINOK(ReadStream_FALSE(stream, h, kStartSize)); + if (h[0] != 'M' || h[1] != 'Z') + return S_FALSE; + /* most of PE files contain 0x0090 at offset 2. + But some rare PE files contain another values. So we don't use that check. + if (Get16(h + 2) != 0x90) return false; */ + _peOffset = Get32(h + 0x3C); + if (!CheckPeOffset(_peOffset)) + return S_FALSE; + coffOffset = _peOffset + 4; + } + { + Byte h[kPeHeaderSize]; + RINOK(stream->Seek(_peOffset, STREAM_SEEK_SET, NULL)); + RINOK(ReadStream_FALSE(stream, h, kPeHeaderSize)); + if (!_header.ParsePe(h)) + return S_FALSE; + } } - UInt32 bufSize = _header.OptHeaderSize + (UInt32)_header.NumSections * kSectionSize; - _totalSize = _peOffset + kHeaderSize + bufSize; + const UInt32 optStart = coffOffset + kCoffHeaderSize; + const UInt32 bufSize = _header.OptHeaderSize + (UInt32)_header.NumSections * kSectionSize; + _totalSize = optStart + bufSize; CByteBuffer buffer(bufSize); RINOK(ReadStream_FALSE(stream, buffer, bufSize)); + + if (_header.OptHeaderSize != 0) if (!_optHeader.Parse(buffer, _header.OptHeaderSize)) return S_FALSE; @@ -2207,7 +2324,9 @@ HRESULT CHandler::Open2(IInStream *stream, IArchiveOpenCallback *callback) for (i = 0; i < _sections.Size(); i++) _sections[i].UpdateTotalSize(_totalSize); - bool thereISDebug; + bool thereISDebug = false; + if (IsOpt()) + { RINOK(LoadDebugSections(stream, thereISDebug)); const CDirLink &certLink = _optHeader.DirItems[kDirLink_Certificate]; @@ -2256,8 +2375,9 @@ HRESULT CHandler::Open2(IInStream *stream, IArchiveOpenCallback *callback) _totalSize += (UInt32)k; } } + } - if (_header.NumSymbols > 0 && _header.PointerToSymbolTable >= 512) + if (_header.NumSymbols > 0 && _header.PointerToSymbolTable >= optStart) { if (_header.NumSymbols >= (1 << 24)) return S_FALSE; @@ -2306,11 +2426,12 @@ HRESULT CHandler::Open2(IInStream *stream, IArchiveOpenCallback *callback) } + if (IsOpt()) if (_optHeader.CheckSum != 0) { RINOK(stream->Seek(0, STREAM_SEEK_SET, NULL)); UInt32 checkSum = 0; - RINOK(CalcCheckSum(stream, _totalSize, _peOffset + kHeaderSize + k_CheckSum_Field_Offset, checkSum)); + RINOK(CalcCheckSum(stream, _totalSize, optStart + k_CheckSum_Field_Offset, checkSum)); _checksumError = (checkSum != _optHeader.CheckSum); } @@ -2333,6 +2454,7 @@ HRESULT CHandler::Open2(IInStream *stream, IArchiveOpenCallback *callback) const CSection § = _sections[i]; CMixItem mixItem; mixItem.SectionIndex = i; + if (IsOpt()) if (_parseResources && sect.Name == ".rsrc" && _items.IsEmpty()) { const unsigned numMixItems = _mixItems.Size(); @@ -2480,6 +2602,8 @@ STDMETHODIMP CHandler::Close() { _totalSize = 0; _checksumError = false; + _mainSubfile = -1; + _stream.Release(); _sections.Clear(); _mixItems.Clear(); @@ -2675,10 +2799,41 @@ REGISTER_ARC_I( 0, NArcInfoFlags::kPreArc, IsArc_Pe) - + } +namespace NCoff { +API_FUNC_static_IsArc IsArc_Coff(const Byte *p, size_t size) +{ + if (size < NPe::kCoffHeaderSize) + return k_IsArc_Res_NEED_MORE; + NPe::CHeader header; + if (!header.ParseCoff(p)) + return k_IsArc_Res_NO; + return k_IsArc_Res_YES; +} +} + +/* +static const Byte k_Signature[] = +{ + 2, 0x4C, 0x01, // x86 + 2, 0x64, 0x86, // x64 + 2, 0x64, 0xAA // ARM64 +}; +REGISTER_ARC_I_CLS( +*/ + +REGISTER_ARC_I_CLS_NO_SIG( + NPe::CHandler(true), + "COFF", "obj", 0, 0xC6, + // k_Signature, + 0, + // NArcInfoFlags::kMultiSignature | + NArcInfoFlags::kStartOpen, + IsArc_Coff) +} namespace NTe { diff --git a/CPP/7zip/Archive/Rar/Rar5Handler.cpp b/CPP/7zip/Archive/Rar/Rar5Handler.cpp index 637af8ce..61c9fc72 100644 --- a/CPP/7zip/Archive/Rar/Rar5Handler.cpp +++ b/CPP/7zip/Archive/Rar/Rar5Handler.cpp @@ -139,11 +139,10 @@ static unsigned ReadVarInt(const Byte *p, size_t maxSize, UInt64 *val) { *val = 0; - for (unsigned i = 0; i < maxSize;) + for (unsigned i = 0; i < maxSize && i < 10;) { Byte b = p[i]; - if (i < 10) - *val |= (UInt64)(b & 0x7F) << (7 * i); + *val |= (UInt64)(b & 0x7F) << (7 * i); i++; if ((b & 0x80) == 0) return i; @@ -1086,7 +1085,7 @@ HRESULT CUnpacker::Create(DECL_EXTERNAL_CODECS_LOC_VARS const CItem &item, bool if (!lzCoder) { const UInt32 methodID = 0x40305; - RINOK(CreateCoder(EXTERNAL_CODECS_LOC_VARS methodID, false, lzCoder)); + RINOK(CreateCoder_Id(EXTERNAL_CODECS_LOC_VARS methodID, false, lzCoder)); if (!lzCoder) return E_NOTIMPL; } @@ -1363,7 +1362,9 @@ static const Byte kProps[] = kpidCharacts, kpidSymLink, kpidHardLink, - kpidCopyLink + kpidCopyLink, + + kpidVolumeIndex }; @@ -1794,6 +1795,18 @@ STDMETHODIMP CHandler::GetProperty(UInt32 index, PROPID propID, PROPVARIANT *val case kpidSplitBefore: prop = item.IsSplitBefore(); break; case kpidSplitAfter: prop = lastItem.IsSplitAfter(); break; + + case kpidVolumeIndex: + { + if (item.VolIndex < _arcs.Size()) + { + const CInArcInfo &arcInfo = _arcs[item.VolIndex].Info; + if (arcInfo.IsVolume()) + prop = (UInt64)arcInfo.GetVolIndex(); + } + break; + } + case kpidCRC: { const CItem *item2 = (lastItem.IsSplitAfter() ? &item : &lastItem); diff --git a/CPP/7zip/Archive/Rar/Rar5Handler.h b/CPP/7zip/Archive/Rar/Rar5Handler.h index bfdbc7b4..c93e8fba 100644 --- a/CPP/7zip/Archive/Rar/Rar5Handler.h +++ b/CPP/7zip/Archive/Rar/Rar5Handler.h @@ -180,7 +180,7 @@ struct CItem AString Name; - int VolIndex; + unsigned VolIndex; int NextItem; UInt32 UnixMTime; diff --git a/CPP/7zip/Archive/Rar/RarHandler.cpp b/CPP/7zip/Archive/Rar/RarHandler.cpp index dcce425d..e23c4a96 100644 --- a/CPP/7zip/Archive/Rar/RarHandler.cpp +++ b/CPP/7zip/Archive/Rar/RarHandler.cpp @@ -777,7 +777,9 @@ static const Byte kProps[] = kpidCRC, kpidHostOS, kpidMethod, - kpidUnpackVer + kpidUnpackVer, + + kpidVolumeIndex }; static const Byte kArcProps[] = @@ -995,6 +997,12 @@ STDMETHODIMP CHandler::GetProperty(UInt32 index, PROPID propID, PROPVARIANT *val case kpidCommented: prop = item.IsCommented(); break; case kpidSplitBefore: prop = item.IsSplitBefore(); break; case kpidSplitAfter: prop = _items[refItem.ItemIndex + refItem.NumItems - 1].IsSplitAfter(); break; + + case kpidVolumeIndex: + if (_arcInfo.Is_VolNumber_Defined()) + prop = (UInt32)(_arcInfo.VolNumber + refItem.VolumeIndex); + break; + case kpidCRC: { prop = ((lastItem.IsSplitAfter()) ? item.FileCRC : lastItem.FileCRC); @@ -1691,7 +1699,7 @@ STDMETHODIMP CHandler::Extract(const UInt32 *indices, UInt32 numItems, methodID += 2; else methodID += 3; - RINOK(CreateCoder(EXTERNAL_CODECS_VARS methodID, false, mi.Coder)); + RINOK(CreateCoder_Id(EXTERNAL_CODECS_VARS methodID, false, mi.Coder)); } if (mi.Coder == 0) diff --git a/CPP/7zip/Archive/Rar/RarHandler.h b/CPP/7zip/Archive/Rar/RarHandler.h index 2a6f7bd1..f9384866 100644 --- a/CPP/7zip/Archive/Rar/RarHandler.h +++ b/CPP/7zip/Archive/Rar/RarHandler.h @@ -26,7 +26,7 @@ struct CInArcInfo UInt32 DataCRC; bool EndOfArchive_was_Read; - CInArcInfo(): EndFlags(0), EndOfArchive_was_Read(false) {} + CInArcInfo(): EndFlags(0), EndOfArchive_was_Read(false), VolNumber(0) {} UInt64 GetPhySize() const { return EndPos - StartPos; } diff --git a/CPP/7zip/Archive/SquashfsHandler.cpp b/CPP/7zip/Archive/SquashfsHandler.cpp index 77cd64a7..b1d7a24c 100644 --- a/CPP/7zip/Archive/SquashfsHandler.cpp +++ b/CPP/7zip/Archive/SquashfsHandler.cpp @@ -2,7 +2,6 @@ #include "StdAfx.h" -#include "../../../C/7zCrc.h" #include "../../../C/Alloc.h" #include "../../../C/CpuArch.h" #include "../../../C/Xz.h" @@ -1209,8 +1208,10 @@ HRESULT CHandler::Decompress(ISequentialOutStream *outStream, Byte *outBuf, bool else { ECoderStatus status; - XzUnpacker_Init(&_xz); - SRes res = XzUnpacker_Code(&_xz, dest, &destLen, _inputBuffer, &srcLen, CODER_FINISH_END, &status); + SRes res = XzUnpacker_CodeFull(&_xz, + dest, &destLen, + _inputBuffer, &srcLen, + CODER_FINISH_END, &status); if (res != 0) return SResToHRESULT(res); if (status != CODER_STATUS_NEEDS_MORE_INPUT || !XzUnpacker_IsStreamWasFinished(&_xz)) diff --git a/CPP/7zip/Archive/Tar/TarIn.cpp b/CPP/7zip/Archive/Tar/TarIn.cpp index a830a0da..a3318685 100644 --- a/CPP/7zip/Archive/Tar/TarIn.cpp +++ b/CPP/7zip/Archive/Tar/TarIn.cpp @@ -94,7 +94,9 @@ static bool ParseInt64(const char *p, Int64 &val) static bool ParseInt64_MTime(const char *p, Int64 &val) { - // rare case tar contains spaces instead of MTime + // rare case tar : ZEROs in Docker-Windows TARs + // rare case tar : spaces + if (GetUi32(p) != 0) for (unsigned i = 0; i < 12; i++) if (p[i] != ' ') return ParseInt64(p, val); diff --git a/CPP/7zip/Archive/XzHandler.cpp b/CPP/7zip/Archive/XzHandler.cpp index 59d1d753..74cbca9d 100644 --- a/CPP/7zip/Archive/XzHandler.cpp +++ b/CPP/7zip/Archive/XzHandler.cpp @@ -24,9 +24,7 @@ #include "IArchive.h" -#ifndef EXTRACT_ONLY #include "Common/HandlerOut.h" -#endif using namespace NWindows; @@ -49,14 +47,22 @@ class CHandler: public IInArchive, public IArchiveOpenSeq, public IInArchiveGetStream, + public ISetProperties, + #ifndef EXTRACT_ONLY public IOutArchive, - public ISetProperties, - public CMultiMethodProps, #endif - public CMyUnknownImp + + public CMyUnknownImp, + + #ifndef EXTRACT_ONLY + public CMultiMethodProps + #else + public CCommonMethodProps + #endif { - NCompress::NXz::CStatInfo _stat; + CXzStatInfo _stat; + SRes MainDecodeSRes; bool _isArc; bool _needSeekToStart; @@ -71,34 +77,48 @@ class CHandler: UInt64 _numSolidBytes; - HRESULT SetSolidFromString(const UString &s); - HRESULT SetSolidFromPROPVARIANT(const PROPVARIANT &value); - HRESULT SetProperty(const wchar_t *name, const PROPVARIANT &value); - - void InitSolid() + void InitXz() { + _filterId = 0; _numSolidBytes = XZ_PROPS__BLOCK_SIZE__AUTO; } + #endif + void Init() { - InitSolid(); - _filterId = 0; - CMultiMethodProps::Init(); + #ifndef EXTRACT_ONLY + InitXz(); + CMultiMethodProps::Init(); + #else + CCommonMethodProps::InitCommon(); + #endif } - #endif + HRESULT SetProperty(const wchar_t *name, const PROPVARIANT &value); HRESULT Open2(IInStream *inStream, /* UInt32 flags, */ IArchiveOpenCallback *callback); - HRESULT Decode2(ISequentialInStream *seqInStream, ISequentialOutStream *outStream, - NCompress::NXz::CDecoder &decoder, ICompressProgressInfo *progress) + HRESULT Decode(NCompress::NXz::CDecoder &decoder, + ISequentialInStream *seqInStream, + ISequentialOutStream *outStream, + ICompressProgressInfo *progress) { + #ifndef _7ZIP_ST + decoder._numThreads = _numThreads; + #endif + decoder._memUsage = _memUsage; + + MainDecodeSRes = SZ_OK; + RINOK(decoder.Decode(seqInStream, outStream, NULL, // *outSizeLimit true, // finishStream progress)); - _stat = decoder; + + _stat = decoder.Stat; + MainDecodeSRes = decoder.MainDecodeSRes; + _phySize_Defined = true; return S_OK; } @@ -107,9 +127,9 @@ public: MY_QUERYINTERFACE_BEGIN2(IInArchive) MY_QUERYINTERFACE_ENTRY(IArchiveOpenSeq) MY_QUERYINTERFACE_ENTRY(IInArchiveGetStream) + MY_QUERYINTERFACE_ENTRY(ISetProperties) #ifndef EXTRACT_ONLY MY_QUERYINTERFACE_ENTRY(IOutArchive) - MY_QUERYINTERFACE_ENTRY(ISetProperties) #endif MY_QUERYINTERFACE_END MY_ADDREF_RELEASE @@ -117,10 +137,10 @@ public: INTERFACE_IInArchive(;) STDMETHOD(OpenSeq)(ISequentialInStream *stream); STDMETHOD(GetStream)(UInt32 index, ISequentialInStream **stream); + STDMETHOD(SetProperties)(const wchar_t * const *names, const PROPVARIANT *values, UInt32 numProps); #ifndef EXTRACT_ONLY INTERFACE_IOutArchive(;) - STDMETHOD(SetProperties)(const wchar_t * const *names, const PROPVARIANT *values, UInt32 numProps); #endif size_t _blocksArraySize; @@ -146,7 +166,7 @@ CHandler::CHandler(): _blocksArraySize(0) { #ifndef EXTRACT_ONLY - Init(); + InitXz(); #endif } @@ -307,7 +327,7 @@ STDMETHODIMP CHandler::GetArchiveProperty(PROPID propID, PROPVARIANT *value) NCOM::CPropVariant prop; switch (propID) { - case kpidPhySize: if (_phySize_Defined) prop = _stat.PhySize; break; + case kpidPhySize: if (_phySize_Defined) prop = _stat.InSize; break; case kpidNumStreams: if (_stat.NumStreams_Defined) prop = _stat.NumStreams; break; case kpidNumBlocks: if (_stat.NumBlocks_Defined) prop = _stat.NumBlocks; break; case kpidUnpackSize: if (_stat.UnpackSize_Defined) prop = _stat.OutSize; break; @@ -330,19 +350,22 @@ STDMETHODIMP CHandler::GetArchiveProperty(PROPID propID, PROPVARIANT *value) case kpidErrorFlags: { UInt32 v = 0; + SRes sres = MainDecodeSRes; // _stat.DecodeRes2; // if (!_isArc) v |= kpv_ErrorFlags_IsNotArc; - if (_stat.UnexpectedEnd) v |= kpv_ErrorFlags_UnexpectedEnd; + if (/*_stat.UnexpectedEnd */ sres == SZ_ERROR_INPUT_EOF) v |= kpv_ErrorFlags_UnexpectedEnd; if (_stat.DataAfterEnd) v |= kpv_ErrorFlags_DataAfterEnd; - if (_stat.HeadersError) v |= kpv_ErrorFlags_HeadersError; - if (_stat.Unsupported) v |= kpv_ErrorFlags_UnsupportedMethod; - if (_stat.DataError) v |= kpv_ErrorFlags_DataError; - if (_stat.CrcError) v |= kpv_ErrorFlags_CrcError; - prop = v; + if (/* _stat.HeadersError */ sres == SZ_ERROR_ARCHIVE) v |= kpv_ErrorFlags_HeadersError; + if (/* _stat.Unsupported */ sres == SZ_ERROR_UNSUPPORTED) v |= kpv_ErrorFlags_UnsupportedMethod; + if (/* _stat.DataError */ sres == SZ_ERROR_DATA) v |= kpv_ErrorFlags_DataError; + if (/* _stat.CrcError */ sres == SZ_ERROR_CRC) v |= kpv_ErrorFlags_CrcError; + if (v != 0) + prop = v; break; } case kpidMainSubfile: { + // debug only, comment it: // if (_blocks) prop = (UInt32)0; break; } @@ -365,7 +388,7 @@ STDMETHODIMP CHandler::GetProperty(UInt32, PROPID propID, PROPVARIANT *value) switch (propID) { case kpidSize: if (_stat.UnpackSize_Defined) prop = _stat.OutSize; break; - case kpidPackSize: if (_phySize_Defined) prop = _stat.PhySize; break; + case kpidPackSize: if (_phySize_Defined) prop = _stat.InSize; break; case kpidMethod: if (!_methodsString.IsEmpty()) prop = _methodsString; break; } prop.Detach(value); @@ -483,10 +506,10 @@ HRESULT CHandler::Open2(IInStream *inStream, /* UInt32 flags, */ IArchiveOpenCal } } - RINOK(inStream->Seek(0, STREAM_SEEK_END, &_stat.PhySize)); + RINOK(inStream->Seek(0, STREAM_SEEK_END, &_stat.InSize)); if (callback) { - RINOK(callback->SetTotal(NULL, &_stat.PhySize)); + RINOK(callback->SetTotal(NULL, &_stat.InSize)); } CSeekInStreamWrap inStreamImp; @@ -621,7 +644,7 @@ STDMETHODIMP CHandler::OpenSeq(ISequentialInStream *stream) STDMETHODIMP CHandler::Close() { - _stat.Clear(); + XzStatInfo_Clear(&_stat); _isArc = false; _needSeekToStart = false; @@ -637,6 +660,8 @@ STDMETHODIMP CHandler::Close() _blocksArraySize = 0; _maxBlocksSize = 0; + MainDecodeSRes = SZ_OK; + return S_OK; } @@ -743,6 +768,8 @@ static HRESULT DecodeBlock(CXzUnpackerCPP2 &xzu, xzu.p.streamFlags = (UInt16)streamFlags; XzUnpacker_PrepareToRandomBlockDecoding(&xzu.p); + XzUnpacker_SetOutBuf(&xzu.p, dest, unpackSize); + const UInt64 packSizeAligned = packSize + ((0 - (unsigned)packSize) & 3); UInt64 packRem = packSizeAligned; @@ -771,8 +798,11 @@ static HRESULT DecodeBlock(CXzUnpackerCPP2 &xzu, ECoderStatus status; SRes res = XzUnpacker_Code(&xzu.p, - dest + outPos, &outLen, + // dest + outPos, + NULL, + &outLen, xzu.InBuf + inPos, &inLen, + (inLen == 0), // srcFinished CODER_FINISH_END, &status); // return E_OUTOFMEMORY; @@ -890,15 +920,16 @@ STDMETHODIMP CHandler::GetStream(UInt32 index, ISequentialInStream **stream) return E_INVALIDARG; if (!_stat.UnpackSize_Defined + || _maxBlocksSize == 0 // 18.02 || _maxBlocksSize > kMaxBlockSize_for_GetStream || _maxBlocksSize != (size_t)_maxBlocksSize) return S_FALSE; - UInt64 physSize = (UInt64)(sizeof(size_t)) << 29; - bool ramSize_Defined = NSystem::GetRamSize(physSize); - if (ramSize_Defined) + UInt64 memSize; + if (!NSystem::GetRamSize(memSize)) + memSize = (UInt64)(sizeof(size_t)) << 28; { - if (_maxBlocksSize > physSize / 4) + if (_maxBlocksSize > memSize / 4) return S_FALSE; } @@ -917,6 +948,31 @@ STDMETHODIMP CHandler::GetStream(UInt32 index, ISequentialInStream **stream) } +static Int32 Get_Extract_OperationResult(const NCompress::NXz::CDecoder &decoder) +{ + Int32 opRes; + SRes sres = decoder.MainDecodeSRes; // decoder.Stat.DecodeRes2; + if (sres == SZ_ERROR_NO_ARCHIVE) // (!IsArc) + opRes = NExtract::NOperationResult::kIsNotArc; + else if (sres == SZ_ERROR_INPUT_EOF) // (UnexpectedEnd) + opRes = NExtract::NOperationResult::kUnexpectedEnd; + else if (decoder.Stat.DataAfterEnd) + opRes = NExtract::NOperationResult::kDataAfterEnd; + else if (sres == SZ_ERROR_CRC) // (CrcError) + opRes = NExtract::NOperationResult::kCRCError; + else if (sres == SZ_ERROR_UNSUPPORTED) // (Unsupported) + opRes = NExtract::NOperationResult::kUnsupportedMethod; + else if (sres == SZ_ERROR_ARCHIVE) // (HeadersError) + opRes = NExtract::NOperationResult::kDataError; + else if (sres == SZ_ERROR_DATA) // (DataError) + opRes = NExtract::NOperationResult::kDataError; + else if (sres != SZ_OK) + opRes = NExtract::NOperationResult::kDataError; + else + opRes = NExtract::NOperationResult::kOK; + return opRes; +} + @@ -930,7 +986,7 @@ STDMETHODIMP CHandler::Extract(const UInt32 *indices, UInt32 numItems, return E_INVALIDARG; if (_phySize_Defined) - extractCallback->SetTotal(_stat.PhySize); + extractCallback->SetTotal(_stat.InSize); UInt64 currentTotalPacked = 0; RINOK(extractCallback->SetCompleted(¤tTotalPacked)); @@ -959,9 +1015,18 @@ STDMETHODIMP CHandler::Extract(const UInt32 *indices, UInt32 numItems, else _needSeekToStart = true; + NCompress::NXz::CDecoder decoder; - RINOK(Decode2(_seqStream, realOutStream, decoder, lpsRef)); - Int32 opRes = decoder.Get_Extract_OperationResult(); + + HRESULT hres = Decode(decoder, _seqStream, realOutStream, lpsRef); + + if (!decoder.MainDecodeSRes_wasUsed) + return hres == S_OK ? E_FAIL : hres; + + Int32 opRes = Get_Extract_OperationResult(decoder); + if (opRes == NExtract::NOperationResult::kOK + && hres != S_OK) + opRes = NExtract::NOperationResult::kDataError; realOutStream.Release(); return extractCallback->SetOperationResult(opRes); @@ -1112,7 +1177,7 @@ STDMETHODIMP CHandler::UpdateItems(ISequentialOutStream *outStream, UInt32 numIt if (_stream) { if (_phySize_Defined) - RINOK(updateCallback->SetTotal(_stat.PhySize)); + RINOK(updateCallback->SetTotal(_stat.InSize)); RINOK(_stream->Seek(0, STREAM_SEEK_SET, NULL)); } @@ -1125,55 +1190,7 @@ STDMETHODIMP CHandler::UpdateItems(ISequentialOutStream *outStream, UInt32 numIt COM_TRY_END } - -HRESULT CHandler::SetSolidFromString(const UString &s) -{ - UString s2 = s; - s2.MakeLower_Ascii(); - - { - const wchar_t *start = ((const wchar_t *)s2); - const wchar_t *end; - UInt64 v = ConvertStringToUInt64(start, &end); - if (start == end) - return E_INVALIDARG; - if ((unsigned)(end - start) + 1 != s2.Len()) - return E_INVALIDARG; - wchar_t c = *end; - { - unsigned numBits; - switch (c) - { - case 'b': numBits = 0; break; - case 'k': numBits = 10; break; - case 'm': numBits = 20; break; - case 'g': numBits = 30; break; - case 't': numBits = 40; break; - default: return E_INVALIDARG; - } - _numSolidBytes = (v << numBits); - } - } - return S_OK; -} - - -HRESULT CHandler::SetSolidFromPROPVARIANT(const PROPVARIANT &value) -{ - bool isSolid; - switch (value.vt) - { - case VT_EMPTY: isSolid = true; break; - case VT_BOOL: isSolid = (value.boolVal != VARIANT_FALSE); break; - case VT_BSTR: - if (StringToBool(value.bstrVal, isSolid)) - break; - return SetSolidFromString(value.bstrVal); - default: return E_INVALIDARG; - } - _numSolidBytes = (isSolid ? XZ_PROPS__BLOCK_SIZE__SOLID : XZ_PROPS__BLOCK_SIZE__AUTO); - return S_OK; -} +#endif HRESULT CHandler::SetProperty(const wchar_t *nameSpec, const PROPVARIANT &value) @@ -1183,20 +1200,53 @@ HRESULT CHandler::SetProperty(const wchar_t *nameSpec, const PROPVARIANT &value) if (name.IsEmpty()) return E_INVALIDARG; + #ifndef EXTRACT_ONLY + if (name[0] == L's') { - name.Delete(0); - if (name.IsEmpty()) - return SetSolidFromPROPVARIANT(value); - if (value.vt != VT_EMPTY) - return E_INVALIDARG; - return SetSolidFromString(name); + const wchar_t *s = name.Ptr(1); + if (*s == 0) + { + bool useStr = false; + bool isSolid; + switch (value.vt) + { + case VT_EMPTY: isSolid = true; break; + case VT_BOOL: isSolid = (value.boolVal != VARIANT_FALSE); break; + case VT_BSTR: + if (!StringToBool(value.bstrVal, isSolid)) + useStr = true; + break; + default: return E_INVALIDARG; + } + if (!useStr) + { + _numSolidBytes = (isSolid ? XZ_PROPS__BLOCK_SIZE__SOLID : XZ_PROPS__BLOCK_SIZE__AUTO); + return S_OK; + } + } + return ParseSizeString(s, value, + 0, // percentsBase + _numSolidBytes) ? S_OK: E_INVALIDARG; } - + return CMultiMethodProps::SetProperty(name, value); + + #else + + { + HRESULT hres; + if (SetCommonProperty(name, value, hres)) + return hres; + } + + return E_INVALIDARG; + + #endif } + STDMETHODIMP CHandler::SetProperties(const wchar_t * const *names, const PROPVARIANT *values, UInt32 numProps) { COM_TRY_BEGIN @@ -1208,6 +1258,8 @@ STDMETHODIMP CHandler::SetProperties(const wchar_t * const *names, const PROPVAR RINOK(SetProperty(names[i], values[i])); } + #ifndef EXTRACT_ONLY + if (!_filterMethod.MethodName.IsEmpty()) { unsigned k; @@ -1238,12 +1290,13 @@ STDMETHODIMP CHandler::SetProperties(const wchar_t * const *names, const PROPVAR return E_INVALIDARG; } + #endif + return S_OK; COM_TRY_END } -#endif REGISTER_ARC_IO( "xz", "xz txz", "* .tar", 0xC, diff --git a/CPP/7zip/Archive/Zip/ZipAddCommon.cpp b/CPP/7zip/Archive/Zip/ZipAddCommon.cpp index a0beab26..a8365db6 100644 --- a/CPP/7zip/Archive/Zip/ZipAddCommon.cpp +++ b/CPP/7zip/Archive/Zip/ZipAddCommon.cpp @@ -384,7 +384,7 @@ HRESULT CAddCommon::Compress( methodId = kMethodId_ZipBase + method; break; } - RINOK(CreateCoder( + RINOK(CreateCoder_Id( EXTERNAL_CODECS_LOC_VARS methodId, true, _compressEncoder)); if (!_compressEncoder) diff --git a/CPP/7zip/Archive/Zip/ZipHandler.cpp b/CPP/7zip/Archive/Zip/ZipHandler.cpp index fda0250c..528313a7 100644 --- a/CPP/7zip/Archive/Zip/ZipHandler.cpp +++ b/CPP/7zip/Archive/Zip/ZipHandler.cpp @@ -112,7 +112,8 @@ static const CUInt32PCharPair g_HeaderCharacts[] = { 3, "Descriptor" }, // { 5, "Patched" }, { 6, kMethod_StrongCrypto }, - { 11, "UTF8" } + { 11, "UTF8" }, + { 14, "Alt" } }; struct CIdToNamePair @@ -169,6 +170,7 @@ static const Byte kProps[] = kpidUnpackVer, kpidVolumeIndex, kpidOffset + // kpidIsAltStream }; static const Byte kArcProps[] = @@ -307,6 +309,8 @@ STDMETHODIMP CHandler::GetArchiveProperty(PROPID propID, PROPVARIANT *value) prop = true; break; } + + // case kpidIsAltStream: prop = true; break; } prop.Detach(value); COM_TRY_END @@ -333,6 +337,21 @@ STDMETHODIMP CHandler::GetProperty(UInt32 index, PROPID propID, PROPVARIANT *val UString res; item.GetUnicodeString(res, item.Name, false, _forceCodePage, _specifiedCodePage); NItemName::ReplaceToOsSlashes_Remove_TailSlash(res); + /* + if (item.ParentOfAltStream >= 0) + { + const CItemEx &prevItem = m_Items[item.ParentOfAltStream]; + UString prevName; + prevItem.GetUnicodeString(prevName, prevItem.Name, false, _forceCodePage, _specifiedCodePage); + NItemName::ReplaceToOsSlashes_Remove_TailSlash(prevName); + if (res.IsPrefixedBy(prevName)) + if (IsString1PrefixedByString2(res.Ptr(prevName.Len()), k_SpecName_NTFS_STREAM)) + { + res.Delete(prevName.Len(), (unsigned)strlen(k_SpecName_NTFS_STREAM)); + res.Insert(prevName.Len(), L":"); + } + } + */ prop = res; break; } @@ -596,6 +615,19 @@ STDMETHODIMP CHandler::GetProperty(UInt32 index, PROPID propID, PROPVARIANT *val case kpidOffset: prop = item.LocalHeaderPos; break; + + /* + case kpidIsAltStream: + prop = (bool)(item.ParentOfAltStream >= 0); // item.IsAltStream(); + break; + + case kpidName: + if (item.ParentOfAltStream >= 0) + { + // extract name of stream here + } + break; + */ } prop.Detach(value); @@ -604,6 +636,85 @@ STDMETHODIMP CHandler::GetProperty(UInt32 index, PROPID propID, PROPVARIANT *val } + +/* +STDMETHODIMP CHandler::GetNumRawProps(UInt32 *numProps) +{ + *numProps = 0; + return S_OK; +} + +STDMETHODIMP CHandler::GetRawPropInfo(UInt32 index, BSTR *name, PROPID *propID) +{ + UNUSED_VAR(index); + *propID = 0; + *name = 0; + return S_OK; +} + +STDMETHODIMP CHandler::GetParent(UInt32 index, UInt32 *parent, UInt32 *parentType) +{ + *parentType = NParentType::kDir; + *parent = (UInt32)(Int32)-1; + if (index >= m_Items.Size()) + return S_OK; + const CItemEx &item = m_Items[index]; + + if (item.ParentOfAltStream >= 0) + { + *parentType = NParentType::kAltStream; + *parent = item.ParentOfAltStream; + } + return S_OK; +} + +STDMETHODIMP CHandler::GetRawProp(UInt32 index, PROPID propID, const void **data, UInt32 *dataSize, UInt32 *propType) +{ + UNUSED_VAR(index); + UNUSED_VAR(propID); + *data = NULL; + *dataSize = 0; + *propType = 0; + return S_OK; +} + + +void CHandler::MarkAltStreams(CObjectVector &items) +{ + int prevIndex = -1; + UString prevName; + UString name; + + for (unsigned i = 0; i < items.Size(); i++) + { + CItemEx &item = m_Items[i]; + if (item.IsAltStream()) + { + if (prevIndex == -1) + continue; + if (prevName.IsEmpty()) + { + const CItemEx &prevItem = m_Items[prevIndex]; + prevItem.GetUnicodeString(prevName, prevItem.Name, false, _forceCodePage, _specifiedCodePage); + NItemName::ReplaceToOsSlashes_Remove_TailSlash(prevName); + } + name.Empty(); + item.GetUnicodeString(name, item.Name, false, _forceCodePage, _specifiedCodePage); + NItemName::ReplaceToOsSlashes_Remove_TailSlash(name); + + if (name.IsPrefixedBy(prevName)) + if (IsString1PrefixedByString2(name.Ptr(prevName.Len()), k_SpecName_NTFS_STREAM)) + item.ParentOfAltStream = prevIndex; + } + else + { + prevIndex = i; + prevName.Empty(); + } + } +} +*/ + STDMETHODIMP CHandler::Open(IInStream *inStream, const UInt64 *maxCheckStartPosition, IArchiveOpenCallback *callback) { @@ -617,6 +728,7 @@ STDMETHODIMP CHandler::Open(IInStream *inStream, m_Items.Clear(); m_Archive.ClearRefs(); // we don't want to clear error flags } + // MarkAltStreams(m_Items); return res; } catch(...) { Close(); throw; } @@ -738,7 +850,7 @@ public: IArchiveExtractCallback *extractCallback, ICompressProgressInfo *compressProgress, #ifndef _7ZIP_ST - UInt32 numThreads, + UInt32 numThreads, UInt64 memUsage, #endif Int32 &res); }; @@ -767,7 +879,7 @@ HRESULT CZipDecoder::Decode( IArchiveExtractCallback *extractCallback, ICompressProgressInfo *compressProgress, #ifndef _7ZIP_ST - UInt32 numThreads, + UInt32 numThreads, UInt64 memUsage, #endif Int32 &res) { @@ -962,7 +1074,7 @@ HRESULT CZipDecoder::Decode( szMethodID = kMethodId_ZipBase + (Byte)id; } - RINOK(CreateCoder(EXTERNAL_CODECS_LOC_VARS szMethodID, false, mi.Coder)); + RINOK(CreateCoder_Id(EXTERNAL_CODECS_LOC_VARS szMethodID, false, mi.Coder)); if (!mi.Coder) { @@ -974,16 +1086,7 @@ HRESULT CZipDecoder::Decode( } ICompressCoder *coder = methodItems[m].Coder; - - { - CMyComPtr setDecoderProperties; - coder->QueryInterface(IID_ICompressSetDecoderProperties2, (void **)&setDecoderProperties); - if (setDecoderProperties) - { - Byte properties = (Byte)item.Flags; - RINOK(setDecoderProperties->SetDecoderProperties2(&properties, 1)); - } - } + #ifndef _7ZIP_ST { @@ -994,7 +1097,27 @@ HRESULT CZipDecoder::Decode( RINOK(setCoderMt->SetNumberOfThreads(numThreads)); } } + // if (memUsage != 0) + { + CMyComPtr setMemLimit; + coder->QueryInterface(IID_ICompressSetMemLimit, (void **)&setMemLimit); + if (setMemLimit) + { + RINOK(setMemLimit->SetMemLimit(memUsage)); + } + } #endif + + { + CMyComPtr setDecoderProperties; + coder->QueryInterface(IID_ICompressSetDecoderProperties2, (void **)&setDecoderProperties); + if (setDecoderProperties) + { + Byte properties = (Byte)item.Flags; + RINOK(setDecoderProperties->SetDecoderProperties2(&properties, 1)); + } + } + CMyComPtr inStreamNew; @@ -1319,7 +1442,7 @@ STDMETHODIMP CHandler::Extract(const UInt32 *indices, UInt32 numItems, m_Archive, item, realOutStream, extractCallback, progress, #ifndef _7ZIP_ST - _props._numThreads, + _props._numThreads, _props._memUsage, #endif res); diff --git a/CPP/7zip/Archive/Zip/ZipHandler.h b/CPP/7zip/Archive/Zip/ZipHandler.h index 4b70a655..404ecc9a 100644 --- a/CPP/7zip/Archive/Zip/ZipHandler.h +++ b/CPP/7zip/Archive/Zip/ZipHandler.h @@ -25,6 +25,7 @@ extern const char * const kMethodNames2[kNumMethodNames2]; class CHandler: public IInArchive, + // public IArchiveGetRawProps, public IOutArchive, public ISetProperties, PUBLIC_ISetCompressCodecsInfo @@ -32,6 +33,7 @@ class CHandler: { public: MY_QUERYINTERFACE_BEGIN2(IInArchive) + // MY_QUERYINTERFACE_ENTRY(IArchiveGetRawProps) MY_QUERYINTERFACE_ENTRY(IOutArchive) MY_QUERYINTERFACE_ENTRY(ISetProperties) QUERY_ENTRY_ISetCompressCodecsInfo @@ -39,6 +41,7 @@ public: MY_ADDREF_RELEASE INTERFACE_IInArchive(;) + // INTERFACE_IArchiveGetRawProps(;) INTERFACE_IOutArchive(;) STDMETHOD(SetProperties)(const wchar_t * const *names, const PROPVARIANT *values, UInt32 numProps); @@ -75,6 +78,10 @@ private: _forceCodePage = false; _specifiedCodePage = CP_OEMCP; } + + // void MarkAltStreams(CObjectVector &items); + + HRESULT GetOutProperty(IArchiveUpdateCallback *callback, UInt32 callbackIndex, Int32 arcIndex, PROPID propID, PROPVARIANT *value); }; }} diff --git a/CPP/7zip/Archive/Zip/ZipHandlerOut.cpp b/CPP/7zip/Archive/Zip/ZipHandlerOut.cpp index 22eb9d13..aec75105 100644 --- a/CPP/7zip/Archive/Zip/ZipHandlerOut.cpp +++ b/CPP/7zip/Archive/Zip/ZipHandlerOut.cpp @@ -136,6 +136,7 @@ STDMETHODIMP CHandler::UpdateItems(ISequentialOutStream *outStream, UInt32 numIt thereAreAesUpdates = true; if (!IntToBool(newProps)) ui.IsDir = inputItem.IsDir(); + // ui.IsAltStream = inputItem.IsAltStream(); } if (IntToBool(newProps)) @@ -175,6 +176,33 @@ STDMETHODIMP CHandler::UpdateItems(ISequentialOutStream *outStream, UInt32 numIt ui.IsDir = (prop.boolVal != VARIANT_FALSE); } + /* + { + bool isAltStream = false; + { + NCOM::CPropVariant prop; + RINOK(callback->GetProperty(i, kpidIsAltStream, &prop)); + if (prop.vt == VT_BOOL) + isAltStream = (prop.boolVal != VARIANT_FALSE); + else if (prop.vt != VT_EMPTY) + return E_INVALIDARG; + } + + if (isAltStream) + { + if (ui.IsDir) + return E_INVALIDARG; + int delim = name.ReverseFind(L':'); + if (delim >= 0) + { + name.Delete(delim, 1); + name.Insert(delim, UString(k_SpecName_NTFS_STREAM)); + ui.IsAltStream = true; + } + } + } + */ + { CPropVariant prop; RINOK(callback->GetProperty(i, kpidTimeType, &prop)); @@ -348,7 +376,8 @@ STDMETHODIMP CHandler::UpdateItems(ISequentialOutStream *outStream, UInt32 numIt { CMethodId methodId; UInt32 numStreams; - if (!FindMethod(EXTERNAL_CODECS_VARS methodName, methodId, numStreams)) + if (FindMethod_Index(EXTERNAL_CODECS_VARS methodName, true, + methodId, numStreams) < 0) return E_NOTIMPL; if (numStreams != 1) return E_NOTIMPL; diff --git a/CPP/7zip/Archive/Zip/ZipHeader.h b/CPP/7zip/Archive/Zip/ZipHeader.h index 58359c2a..f99ae598 100644 --- a/CPP/7zip/Archive/Zip/ZipHeader.h +++ b/CPP/7zip/Archive/Zip/ZipHeader.h @@ -131,6 +131,7 @@ namespace NFileHeader const unsigned kDescriptorUsedMask = 1 << 3; const unsigned kStrongEncrypted = 1 << 6; const unsigned kUtf8 = 1 << 11; + const unsigned kAltStream = 1 << 14; const unsigned kImplodeDictionarySizeMask = 1 << 1; const unsigned kImplodeLiteralsOnMask = 1 << 2; diff --git a/CPP/7zip/Archive/Zip/ZipIn.cpp b/CPP/7zip/Archive/Zip/ZipIn.cpp index 0e4fd240..dc77757e 100644 --- a/CPP/7zip/Archive/Zip/ZipIn.cpp +++ b/CPP/7zip/Archive/Zip/ZipIn.cpp @@ -1993,6 +1993,13 @@ HRESULT CVols::ParseArcName(IArchiveOpenVolumeCallback *volCallback) } else if (ext.IsEqualTo_Ascii_NoCase("exe")) { + /* possible cases: + - exe with zip inside + - sfx: a.exe, a.z02, a.z03,... , a.zip + a.exe is start volume. + - zip renamed to exe + */ + StartIsExe = true; BaseName = name; StartVolIndex = 0; @@ -2000,7 +2007,22 @@ HRESULT CVols::ParseArcName(IArchiveOpenVolumeCallback *volCallback) We can open arc.zip, if it was requesed to open arc.exe. But it's possible that arc.exe and arc.zip are not parts of same archive. So we can disable such operation */ - return S_FALSE; // don't open arc.zip instead of arc.exe + + // 18.04: we still want to open zip renamed to exe. + /* + { + UString volName = name; + volName += IsUpperCase ? "Z01" : "z01"; + { + CMyComPtr stream; + HRESULT res2 = volCallback->GetStream(volName, &stream); + if (res2 == S_OK) + DisableVolsSearch = true; + } + } + */ + DisableVolsSearch = true; + return S_OK; } else if (ext[0] == 'z' || ext[0] == 'Z') { @@ -2040,6 +2062,9 @@ HRESULT CVols::ParseArcName(IArchiveOpenVolumeCallback *volCallback) HRESULT CInArchive::ReadVols2(IArchiveOpenVolumeCallback *volCallback, unsigned start, int lastDisk, int zipDisk, unsigned numMissingVolsMax, unsigned &numMissingVols) { + if (Vols.DisableVolsSearch) + return S_OK; + numMissingVols = 0; for (unsigned i = start;; i++) @@ -2090,6 +2115,8 @@ HRESULT CInArchive::ReadVols2(IArchiveOpenVolumeCallback *volCallback, } if (res == S_FALSE || !stream) { + if (i == 1 && Vols.StartIsExe) + return S_OK; if (Vols.MissingName.IsEmpty()) Vols.MissingName = volName; numMissingVols++; diff --git a/CPP/7zip/Archive/Zip/ZipIn.h b/CPP/7zip/Archive/Zip/ZipIn.h index e51f5ce4..f43f0396 100644 --- a/CPP/7zip/Archive/Zip/ZipIn.h +++ b/CPP/7zip/Archive/Zip/ZipIn.h @@ -20,10 +20,13 @@ class CItemEx: public CItem { public: UInt32 LocalFullHeaderSize; // including Name and Extra + // int ParentOfAltStream; // -1, if not AltStream bool DescriptorWasRead; - CItemEx(): DescriptorWasRead(false) {} + CItemEx(): + // ParentOfAltStream(-1), + DescriptorWasRead(false) {} UInt64 GetLocalFullSize() const { return LocalFullHeaderSize + GetPackSizeWithDescriptor(); } @@ -159,6 +162,7 @@ struct CVols bool NeedSeek; + bool DisableVolsSearch; bool StartIsExe; // is .exe bool StartIsZ; // is .zip or .zNN bool StartIsZip; // is .zip @@ -198,6 +202,7 @@ struct CVols StreamIndex = -1; NeedSeek = false; + DisableVolsSearch = false; StartIsExe = false; StartIsZ = false; StartIsZip = false; diff --git a/CPP/7zip/Archive/Zip/ZipItem.cpp b/CPP/7zip/Archive/Zip/ZipItem.cpp index 330dad40..888f940e 100644 --- a/CPP/7zip/Archive/Zip/ZipItem.cpp +++ b/CPP/7zip/Archive/Zip/ZipItem.cpp @@ -20,6 +20,12 @@ namespace NZip { using namespace NFileHeader; + +/* +const char *k_SpecName_NTFS_STREAM = "@@NTFS@STREAM@"; +const char *k_SpecName_MAC_RESOURCE_FORK = "@@MAC@RESOURCE-FORK@"; +*/ + static const CUInt32PCharPair g_ExtraTypes[] = { { NExtraID::kZip64, "Zip64" }, diff --git a/CPP/7zip/Archive/Zip/ZipItem.h b/CPP/7zip/Archive/Zip/ZipItem.h index d488cbd7..38b44c12 100644 --- a/CPP/7zip/Archive/Zip/ZipItem.h +++ b/CPP/7zip/Archive/Zip/ZipItem.h @@ -14,6 +14,11 @@ namespace NArchive { namespace NZip { +/* +extern const char *k_SpecName_NTFS_STREAM; +extern const char *k_SpecName_MAC_RESOURCE_FORK; +*/ + struct CVersion { Byte Version; @@ -233,6 +238,7 @@ public: bool IsAesEncrypted() const { return IsEncrypted() && (IsStrongEncrypted() || Method == NFileHeader::NCompressionMethod::kWzAES); } bool IsLzmaEOS() const { return (Flags & NFileHeader::NFlags::kLzmaEOS) != 0; } bool HasDescriptor() const { return (Flags & NFileHeader::NFlags::kDescriptorUsedMask) != 0; } + // bool IsAltStream() const { return (Flags & NFileHeader::NFlags::kAltStream) != 0; } unsigned GetDeflateLevel() const { return (Flags >> 1) & 3; } @@ -264,6 +270,7 @@ public: void ClearFlags() { Flags = 0; } void SetEncrypted(bool encrypted) { SetFlag(NFileHeader::NFlags::kEncrypted, encrypted); } void SetUtf8(bool isUtf8) { SetFlag(NFileHeader::NFlags::kUtf8, isUtf8); } + // void SetFlag_AltStream(bool isAltStream) { SetFlag(NFileHeader::NFlags::kAltStream, isAltStream); } void SetDescriptorMode(bool useDescriptor) { SetFlag(NFileHeader::NFlags::kDescriptorUsedMask, useDescriptor); } UINT GetCodePage() const { return CP_OEMCP; } diff --git a/CPP/7zip/Archive/Zip/ZipUpdate.cpp b/CPP/7zip/Archive/Zip/ZipUpdate.cpp index 25518ddc..3ceea97b 100644 --- a/CPP/7zip/Archive/Zip/ZipUpdate.cpp +++ b/CPP/7zip/Archive/Zip/ZipUpdate.cpp @@ -75,6 +75,7 @@ static void SetFileHeader( item.Name = ui.Name; item.Comment = ui.Comment; item.SetUtf8(ui.IsUtf8); + // item.SetFlag_AltStream(ui.IsAltStream); item.ExternalAttrib = ui.Attrib; item.Time = ui.Time; item.Ntfs_MTime = ui.Ntfs_MTime; @@ -280,6 +281,7 @@ public: MY_UNKNOWN_IMP void Create(IProgress *progress, bool inSizeIsMain); void SetProgressOffset(UInt64 progressOffset); + void SetProgressOffset_NoLock(UInt64 progressOffset); HRESULT SetRatioInfo(unsigned index, const UInt64 *inSize, const UInt64 *outSize); STDMETHOD(SetRatioInfo)(const UInt64 *inSize, const UInt64 *outSize); }; @@ -292,11 +294,16 @@ void CMtProgressMixer2::Create(IProgress *progress, bool inSizeIsMain) ProgressOffset = InSizes[0] = InSizes[1] = OutSizes[0] = OutSizes[1] = 0; } +void CMtProgressMixer2::SetProgressOffset_NoLock(UInt64 progressOffset) +{ + InSizes[1] = OutSizes[1] = 0; + ProgressOffset = progressOffset; +} + void CMtProgressMixer2::SetProgressOffset(UInt64 progressOffset) { CriticalSection.Enter(); - InSizes[1] = OutSizes[1] = 0; - ProgressOffset = progressOffset; + SetProgressOffset_NoLock(progressOffset); CriticalSection.Leave(); } @@ -384,6 +391,7 @@ static HRESULT UpdateItemOldData( item.Comment = ui.Comment; item.Name = ui.Name; item.SetUtf8(ui.IsUtf8); + // item.SetFlag_AltStream(ui.IsAltStream); item.Time = ui.Time; item.Ntfs_MTime = ui.Ntfs_MTime; item.Ntfs_ATime = ui.Ntfs_ATime; @@ -602,8 +610,11 @@ static HRESULT Update2St( lps->InSize = unpackSizeTotal; lps->OutSize = packSizeTotal; RINOK(lps->SetCur()); + archive.WriteCentralDir(items, comment); - return S_OK; + + lps->ProgressOffset += kCentralHeaderSize * updateItems.Size() + 1; + return lps->SetCur(); } @@ -897,7 +908,7 @@ static HRESULT Update2( { complexity += ui.Size; complexity += kLocalHeaderSize; - mtProgressMixerSpec->Mixer2->SetProgressOffset(complexity); + mtProgressMixerSpec->Mixer2->SetProgressOffset_NoLock(complexity); RINOK(updateCallback->SetOperationResult(NArchive::NUpdate::NOperationResult::kOK)); memRef2.Skip = true; continue; @@ -1107,8 +1118,13 @@ static HRESULT Update2( } RINOK(mtCompressProgressMixer.SetRatioInfo(0, NULL, NULL)); + archive.WriteCentralDir(items, comment); - return S_OK; + + complexity += kCentralHeaderSize * updateItems.Size() + 1; + mtProgressMixerSpec->Mixer2->SetProgressOffset(complexity); + return mtCompressProgressMixer.SetRatioInfo(0, NULL, NULL); + #endif } diff --git a/CPP/7zip/Archive/Zip/ZipUpdate.h b/CPP/7zip/Archive/Zip/ZipUpdate.h index 0907d9a5..7abb6084 100644 --- a/CPP/7zip/Archive/Zip/ZipUpdate.h +++ b/CPP/7zip/Archive/Zip/ZipUpdate.h @@ -32,6 +32,7 @@ struct CUpdateItem bool IsDir; bool NtfsTimeIsDefined; bool IsUtf8; + // bool IsAltStream; int IndexInArc; int IndexInClient; UInt32 Attrib; @@ -50,12 +51,19 @@ struct CUpdateItem IsDir = false; NtfsTimeIsDefined = false; IsUtf8 = false; + // IsAltStream = false; Size = 0; Name.Empty(); Comment.Free(); } - CUpdateItem(): NtfsTimeIsDefined(false), IsUtf8(false), Size(0) {} + CUpdateItem(): + IsDir(false), + NtfsTimeIsDefined(false), + IsUtf8(false), + // IsAltStream(false), + Size(0) + {} }; HRESULT Update( diff --git a/CPP/7zip/Bundles/Alone/Alone.dsp b/CPP/7zip/Bundles/Alone/Alone.dsp index e40fa7a8..9b5d7531 100644 --- a/CPP/7zip/Bundles/Alone/Alone.dsp +++ b/CPP/7zip/Bundles/Alone/Alone.dsp @@ -1112,18 +1112,6 @@ SOURCE=..\..\Compress\ShrinkDecoder.cpp SOURCE=..\..\Compress\ShrinkDecoder.h # End Source File # End Group -# Begin Group "Z" - -# PROP Default_Filter "" -# Begin Source File - -SOURCE=..\..\Compress\ZDecoder.cpp -# End Source File -# Begin Source File - -SOURCE=..\..\Compress\ZDecoder.h -# End Source File -# End Group # Begin Group "BWT" # PROP Default_Filter "" @@ -1652,10 +1640,6 @@ SOURCE=..\..\Archive\SplitHandler.cpp SOURCE=..\..\Archive\XzHandler.cpp # End Source File -# Begin Source File - -SOURCE=..\..\Archive\ZHandler.cpp -# End Source File # End Group # Begin Group "UI Common" @@ -2636,6 +2620,34 @@ SOURCE=..\..\..\..\C\Lzma2Dec.h # End Source File # Begin Source File +SOURCE=..\..\..\..\C\Lzma2DecMt.c + +!IF "$(CFG)" == "Alone - Win32 Release" + +# ADD CPP /O2 +# SUBTRACT CPP /YX /Yc /Yu + +!ELSEIF "$(CFG)" == "Alone - Win32 Debug" + +# SUBTRACT CPP /YX /Yc /Yu + +!ELSEIF "$(CFG)" == "Alone - Win32 ReleaseU" + +# SUBTRACT CPP /YX /Yc /Yu + +!ELSEIF "$(CFG)" == "Alone - Win32 DebugU" + +# SUBTRACT CPP /YX /Yc /Yu + +!ENDIF + +# End Source File +# Begin Source File + +SOURCE=..\..\..\..\C\Lzma2DecMt.h +# End Source File +# Begin Source File + SOURCE=..\..\..\..\C\Lzma2Enc.c !IF "$(CFG)" == "Alone - Win32 Release" @@ -2748,6 +2760,34 @@ SOURCE=..\..\..\..\C\MtCoder.h # End Source File # Begin Source File +SOURCE=..\..\..\..\C\MtDec.c + +!IF "$(CFG)" == "Alone - Win32 Release" + +# ADD CPP /O2 +# SUBTRACT CPP /YX /Yc /Yu + +!ELSEIF "$(CFG)" == "Alone - Win32 Debug" + +# SUBTRACT CPP /YX /Yc /Yu + +!ELSEIF "$(CFG)" == "Alone - Win32 ReleaseU" + +# SUBTRACT CPP /YX /Yc /Yu + +!ELSEIF "$(CFG)" == "Alone - Win32 DebugU" + +# SUBTRACT CPP /YX /Yc /Yu + +!ENDIF + +# End Source File +# Begin Source File + +SOURCE=..\..\..\..\C\MtDec.h +# End Source File +# Begin Source File + SOURCE=..\..\..\..\C\Ppmd.h # End Source File # Begin Source File diff --git a/CPP/7zip/Bundles/Alone/makefile b/CPP/7zip/Bundles/Alone/makefile index 73cea464..4b55bad8 100644 --- a/CPP/7zip/Bundles/Alone/makefile +++ b/CPP/7zip/Bundles/Alone/makefile @@ -76,7 +76,6 @@ AR_OBJS = \ $O\LzmaHandler.obj \ $O\SplitHandler.obj \ $O\XzHandler.obj \ - $O\ZHandler.obj \ AR_COMMON_OBJS = \ $O\CoderMixer2.obj \ @@ -170,7 +169,6 @@ COMPRESS_OBJS = \ $O\ShrinkDecoder.obj \ $O\XzDecoder.obj \ $O\XzEncoder.obj \ - $O\ZDecoder.obj \ CRYPTO_OBJS = \ $O\7zAes.obj \ @@ -199,10 +197,12 @@ C_OBJS = \ $O\LzFind.obj \ $O\LzFindMt.obj \ $O\Lzma2Dec.obj \ + $O\Lzma2DecMt.obj \ $O\Lzma2Enc.obj \ $O\LzmaDec.obj \ $O\LzmaEnc.obj \ $O\MtCoder.obj \ + $O\MtDec.obj \ $O\Ppmd7.obj \ $O\Ppmd7Dec.obj \ $O\Ppmd7Enc.obj \ @@ -324,5 +324,6 @@ ZSTDMT_OBJS = \ !include "../../Aes.mak" !include "../../Crc.mak" !include "../../Crc64.mak" +!include "../../LzmaDec.mak" !include "../../7zip.mak" diff --git a/CPP/7zip/Bundles/Alone/resource.rc b/CPP/7zip/Bundles/Alone/resource.rc index 9b817070..9b6a3084 100644 --- a/CPP/7zip/Bundles/Alone/resource.rc +++ b/CPP/7zip/Bundles/Alone/resource.rc @@ -1,3 +1,7 @@ #include "../../MyVersionInfo.rc" MY_VERSION_INFO_APP("7-Zip Standalone Console", "7za") + +#ifndef UNDER_CE +1 24 MOVEABLE PURE "../../UI/Console/Console.manifest" +#endif diff --git a/CPP/7zip/Bundles/Alone7z/Alone.dsp b/CPP/7zip/Bundles/Alone7z/Alone.dsp index 10ae3ad2..6147e105 100644 --- a/CPP/7zip/Bundles/Alone7z/Alone.dsp +++ b/CPP/7zip/Bundles/Alone7z/Alone.dsp @@ -44,7 +44,7 @@ RSC=rc.exe # PROP Ignore_Export_Lib 0 # PROP Target_Dir "" # ADD BASE CPP /nologo /W3 /GX /O2 /D "WIN32" /D "NDEBUG" /D "_CONSOLE" /D "_MBCS" /YX /FD /c -# ADD CPP /nologo /Gr /MT /W3 /GX /O1 /I "..\..\..\\" /D "NDEBUG" /D "_MBCS" /D "WIN32" /D "_CONSOLE" /FAc /Yu"StdAfx.h" /FD /c +# ADD CPP /nologo /Gr /MT /W3 /GX /O1 /I "..\..\..\\" /D "NDEBUG" /D "_MBCS" /D "WIN32" /D "_CONSOLE" /D "_7ZIP_LARGE_PAGES" /D "SUPPORT_DEVICE_FILE" /FAc /Yu"StdAfx.h" /FD /c # ADD BASE RSC /l 0x419 /d "NDEBUG" # ADD RSC /l 0x419 /d "NDEBUG" BSC32=bscmake.exe @@ -69,7 +69,7 @@ LINK32=link.exe # PROP Ignore_Export_Lib 0 # PROP Target_Dir "" # ADD BASE CPP /nologo /W3 /Gm /GX /ZI /Od /D "WIN32" /D "_DEBUG" /D "_CONSOLE" /D "_MBCS" /YX /FD /GZ /c -# ADD CPP /nologo /Gr /MDd /W3 /Gm /GX /ZI /Od /I "..\..\..\\" /D "_DEBUG" /D "_MBCS" /D "WIN32" /D "_CONSOLE" /Yu"StdAfx.h" /FD /GZ /c +# ADD CPP /nologo /Gr /MDd /W3 /Gm /GX /ZI /Od /I "..\..\..\\" /D "_DEBUG" /D "_MBCS" /D "WIN32" /D "_CONSOLE" /D "_7ZIP_LARGE_PAGES" /D "SUPPORT_DEVICE_FILE" /Yu"StdAfx.h" /FD /GZ /c # ADD BASE RSC /l 0x419 /d "_DEBUG" # ADD RSC /l 0x419 /d "_DEBUG" BSC32=bscmake.exe @@ -94,7 +94,7 @@ LINK32=link.exe # PROP Ignore_Export_Lib 0 # PROP Target_Dir "" # ADD BASE CPP /nologo /MD /W3 /GX /O2 /D "NDEBUG" /D "WIN32" /D "_CONSOLE" /D "EXCLUDE_COM" /D "NO_REGISTRY" /Yu"StdAfx.h" /FD /c -# ADD CPP /nologo /Gr /MD /W4 /GX /O1 /I "..\..\..\\" /D "NDEBUG" /D "UNICODE" /D "_UNICODE" /D "WIN32" /D "_CONSOLE" /Yu"StdAfx.h" /FD /c +# ADD CPP /nologo /Gr /MD /W4 /GX /O1 /I "..\..\..\\" /D "NDEBUG" /D "UNICODE" /D "_UNICODE" /D "WIN32" /D "_CONSOLE" /D "_7ZIP_LARGE_PAGES" /D "SUPPORT_DEVICE_FILE" /Yu"StdAfx.h" /FD /c # ADD BASE RSC /l 0x419 /d "NDEBUG" # ADD RSC /l 0x419 /d "NDEBUG" BSC32=bscmake.exe @@ -121,7 +121,7 @@ LINK32=link.exe # PROP Ignore_Export_Lib 0 # PROP Target_Dir "" # ADD BASE CPP /nologo /W3 /Gm /GX /ZI /Od /D "_DEBUG" /D "WIN32" /D "_CONSOLE" /D "EXCLUDE_COM" /D "NO_REGISTRY" /D "_MBCS" /Yu"StdAfx.h" /FD /GZ /c -# ADD CPP /nologo /Gr /MDd /W4 /Gm /GX /ZI /Od /I "..\..\..\\" /D "_DEBUG" /D "_UNICODE" /D "UNICODE" /D "WIN32" /D "_CONSOLE" /Yu"StdAfx.h" /FD /GZ /c +# ADD CPP /nologo /Gr /MDd /W4 /Gm /GX /ZI /Od /I "..\..\..\\" /D "_DEBUG" /D "_UNICODE" /D "UNICODE" /D "WIN32" /D "_CONSOLE" /D "_7ZIP_LARGE_PAGES" /D "SUPPORT_DEVICE_FILE" /Yu"StdAfx.h" /FD /GZ /c # ADD BASE RSC /l 0x419 /d "_DEBUG" # ADD RSC /l 0x419 /d "_DEBUG" BSC32=bscmake.exe @@ -482,6 +482,14 @@ SOURCE=..\..\..\Windows\FileName.h # End Source File # Begin Source File +SOURCE=..\..\..\Windows\FileSystem.cpp +# End Source File +# Begin Source File + +SOURCE=..\..\..\Windows\FileSystem.h +# End Source File +# Begin Source File + SOURCE=..\..\..\Windows\Handle.h # End Source File # Begin Source File @@ -1696,6 +1704,34 @@ SOURCE=..\..\..\..\C\Lzma2Dec.h # End Source File # Begin Source File +SOURCE=..\..\..\..\C\Lzma2DecMt.c + +!IF "$(CFG)" == "Alone - Win32 Release" + +# ADD CPP /O2 +# SUBTRACT CPP /YX /Yc /Yu + +!ELSEIF "$(CFG)" == "Alone - Win32 Debug" + +# SUBTRACT CPP /YX /Yc /Yu + +!ELSEIF "$(CFG)" == "Alone - Win32 ReleaseU" + +# SUBTRACT CPP /YX /Yc /Yu + +!ELSEIF "$(CFG)" == "Alone - Win32 DebugU" + +# SUBTRACT CPP /YX /Yc /Yu + +!ENDIF + +# End Source File +# Begin Source File + +SOURCE=..\..\..\..\C\Lzma2DecMt.h +# End Source File +# Begin Source File + SOURCE=..\..\..\..\C\Lzma2Enc.c !IF "$(CFG)" == "Alone - Win32 Release" @@ -1789,6 +1825,34 @@ SOURCE=..\..\..\..\C\MtCoder.h # End Source File # Begin Source File +SOURCE=..\..\..\..\C\MtDec.c + +!IF "$(CFG)" == "Alone - Win32 Release" + +# ADD CPP /O2 +# SUBTRACT CPP /YX /Yc /Yu + +!ELSEIF "$(CFG)" == "Alone - Win32 Debug" + +# SUBTRACT CPP /YX /Yc /Yu + +!ELSEIF "$(CFG)" == "Alone - Win32 ReleaseU" + +# SUBTRACT CPP /YX /Yc /Yu + +!ELSEIF "$(CFG)" == "Alone - Win32 DebugU" + +# SUBTRACT CPP /YX /Yc /Yu + +!ENDIF + +# End Source File +# Begin Source File + +SOURCE=..\..\..\..\C\MtDec.h +# End Source File +# Begin Source File + SOURCE=..\..\..\..\C\Threads.c # SUBTRACT CPP /YX /Yc /Yu # End Source File diff --git a/CPP/7zip/Bundles/Alone7z/makefile b/CPP/7zip/Bundles/Alone7z/makefile index bbab5b4a..91ee1922 100644 --- a/CPP/7zip/Bundles/Alone7z/makefile +++ b/CPP/7zip/Bundles/Alone7z/makefile @@ -3,7 +3,7 @@ MY_CONSOLE = 1 CFLAGS = $(CFLAGS) -DPROG_VARIANT_R !IFNDEF UNDER_CE -CFLAGS = $(CFLAGS) -DWIN_LONG_PATH +CFLAGS = $(CFLAGS) -DWIN_LONG_PATH -D_7ZIP_LARGE_PAGES -DSUPPORT_DEVICE_FILE !ENDIF COMMON_OBJS = \ @@ -33,6 +33,7 @@ WIN_OBJS = \ $O\FileIO.obj \ $O\FileLink.obj \ $O\FileName.obj \ + $O\FileSystem.obj \ $O\MemoryLock.obj \ $O\PropVariant.obj \ $O\PropVariantConv.obj \ @@ -133,10 +134,12 @@ C_OBJS = \ $O\LzFind.obj \ $O\LzFindMt.obj \ $O\Lzma2Dec.obj \ + $O\Lzma2DecMt.obj \ $O\Lzma2Enc.obj \ $O\LzmaDec.obj \ $O\LzmaEnc.obj \ $O\MtCoder.obj \ + $O\MtDec.obj \ $O\Sha256.obj \ $O\Sort.obj \ $O\Threads.obj \ @@ -150,5 +153,6 @@ C_OBJS = \ !include "../../Aes.mak" !include "../../Crc.mak" !include "../../Crc64.mak" +!include "../../LzmaDec.mak" !include "../../7zip.mak" diff --git a/CPP/7zip/Bundles/Alone7z/resource.rc b/CPP/7zip/Bundles/Alone7z/resource.rc index 3be08c9f..36d70e7d 100644 --- a/CPP/7zip/Bundles/Alone7z/resource.rc +++ b/CPP/7zip/Bundles/Alone7z/resource.rc @@ -1,3 +1,7 @@ #include "../../../../C/7zVersion.rc" MY_VERSION_INFO_APP("7-Zip Reduced Standalone Console", "7zr") + +#ifndef UNDER_CE +1 24 MOVEABLE PURE "../../UI/Console/Console.manifest" +#endif diff --git a/CPP/7zip/Bundles/Fm/FM.dsp b/CPP/7zip/Bundles/Fm/FM.dsp index 6fbbbfda..08994110 100644 --- a/CPP/7zip/Bundles/Fm/FM.dsp +++ b/CPP/7zip/Bundles/Fm/FM.dsp @@ -677,6 +677,14 @@ SOURCE=..\..\UI\FileManager\DialogSize.h # End Source File # Begin Source File +SOURCE=..\..\UI\FileManager\EditDialog.cpp +# End Source File +# Begin Source File + +SOURCE=..\..\UI\FileManager\EditDialog.h +# End Source File +# Begin Source File + SOURCE=..\..\UI\FileManager\LinkDialog.cpp # End Source File # Begin Source File @@ -1004,6 +1012,15 @@ SOURCE=..\..\..\..\C\Lzma2Dec.h # End Source File # Begin Source File +SOURCE=..\..\..\..\C\Lzma2DecMt.c +# SUBTRACT CPP /YX /Yc /Yu +# End Source File +# Begin Source File + +SOURCE=..\..\..\..\C\Lzma2DecMt.h +# End Source File +# Begin Source File + SOURCE=..\..\..\..\C\Lzma2Enc.c # SUBTRACT CPP /YX /Yc /Yu # End Source File @@ -1040,6 +1057,15 @@ SOURCE=..\..\..\..\C\MtCoder.h # End Source File # Begin Source File +SOURCE=..\..\..\..\C\MtDec.c +# SUBTRACT CPP /YX /Yc /Yu +# End Source File +# Begin Source File + +SOURCE=..\..\..\..\C\MtDec.h +# End Source File +# Begin Source File + SOURCE=..\..\..\..\C\Sha256.c !IF "$(CFG)" == "FM - Win32 Release" @@ -1549,14 +1575,6 @@ SOURCE=..\..\..\Common\Wildcard.h # PROP Default_Filter "" # Begin Source File -SOURCE=..\..\UI\Common\ArchiveCommandLine.cpp -# End Source File -# Begin Source File - -SOURCE=..\..\UI\Common\ArchiveCommandLine.h -# End Source File -# Begin Source File - SOURCE=..\..\UI\Common\ArchiveExtractCallback.cpp # End Source File # Begin Source File diff --git a/CPP/7zip/Bundles/Fm/makefile b/CPP/7zip/Bundles/Fm/makefile index bcec14e1..01ef54dd 100644 --- a/CPP/7zip/Bundles/Fm/makefile +++ b/CPP/7zip/Bundles/Fm/makefile @@ -48,7 +48,6 @@ WIN_CTRL_OBJS = \ $O\FileStreams.obj \ UI_COMMON_OBJS = \ - $O\ArchiveCommandLine.obj \ $O\ArchiveExtractCallback.obj \ $O\ArchiveName.obj \ $O\ArchiveOpenCallback.obj \ diff --git a/CPP/7zip/Bundles/Format7z/makefile b/CPP/7zip/Bundles/Format7z/makefile index 54386f39..7ecffd0a 100644 --- a/CPP/7zip/Bundles/Format7z/makefile +++ b/CPP/7zip/Bundles/Format7z/makefile @@ -122,10 +122,12 @@ C_OBJS = \ $O\LzFind.obj \ $O\LzFindMt.obj \ $O\Lzma2Dec.obj \ + $O\Lzma2DecMt.obj \ $O\Lzma2Enc.obj \ $O\LzmaDec.obj \ $O\LzmaEnc.obj \ $O\MtCoder.obj \ + $O\MtDec.obj \ $O\Ppmd7.obj \ $O\Ppmd7Dec.obj \ $O\Ppmd7Enc.obj \ @@ -135,6 +137,7 @@ C_OBJS = \ !include "../../Aes.mak" !include "../../Crc.mak" +!include "../../LzmaDec.mak" COMPRESS_OBJS = $(COMPRESS_OBJS) \ $O\BrotliDecoder.obj \ diff --git a/CPP/7zip/Bundles/Format7zExtract/makefile b/CPP/7zip/Bundles/Format7zExtract/makefile index 52b217da..56c83577 100644 --- a/CPP/7zip/Bundles/Format7zExtract/makefile +++ b/CPP/7zip/Bundles/Format7zExtract/makefile @@ -43,6 +43,7 @@ AR_OBJS = \ AR_COMMON_OBJS = \ $O\CoderMixer2.obj \ + $O\HandlerOut.obj \ $O\ItemNameUtils.obj \ $O\OutStreamWithCRC.obj \ $O\ParseProperties.obj \ @@ -99,7 +100,9 @@ C_OBJS = \ $O\CpuArch.obj \ $O\Delta.obj \ $O\Lzma2Dec.obj \ + $O\Lzma2DecMt.obj \ $O\LzmaDec.obj \ + $O\MtDec.obj \ $O\Ppmd7.obj \ $O\Ppmd7Dec.obj \ $O\Sha256.obj \ @@ -198,5 +201,6 @@ ZSTDMT_OBJS = \ !include "../../Aes.mak" !include "../../Crc.mak" +!include "../../LzmaDec.mak" !include "../../7zip.mak" diff --git a/CPP/7zip/Bundles/Format7zExtractR/makefile b/CPP/7zip/Bundles/Format7zExtractR/makefile index d34ab460..3a7f9816 100644 --- a/CPP/7zip/Bundles/Format7zExtractR/makefile +++ b/CPP/7zip/Bundles/Format7zExtractR/makefile @@ -85,9 +85,12 @@ C_OBJS = \ $O\CpuArch.obj \ $O\Delta.obj \ $O\Lzma2Dec.obj \ + $O\Lzma2DecMt.obj \ $O\LzmaDec.obj \ + $O\MtDec.obj \ $O\Threads.obj \ !include "../../Crc.mak" +!include "../../LzmaDec.mak" !include "../../7zip.mak" diff --git a/CPP/7zip/Bundles/Format7zF/Arc.mak b/CPP/7zip/Bundles/Format7zF/Arc.mak index 5e7ba6f8..b78e4c1b 100644 --- a/CPP/7zip/Bundles/Format7zF/Arc.mak +++ b/CPP/7zip/Bundles/Format7zF/Arc.mak @@ -202,6 +202,7 @@ COMPRESS_OBJS = \ $O\DeflateRegister.obj \ $O\DeltaFilter.obj \ $O\ImplodeDecoder.obj \ + $O\LzfseDecoder.obj \ $O\LzhDecoder.obj \ $O\Lzma2Decoder.obj \ $O\Lzma2Encoder.obj \ @@ -265,10 +266,12 @@ C_OBJS = \ $O\LzFind.obj \ $O\LzFindMt.obj \ $O\Lzma2Dec.obj \ + $O\Lzma2DecMt.obj \ $O\Lzma2Enc.obj \ $O\LzmaDec.obj \ $O\LzmaEnc.obj \ $O\MtCoder.obj \ + $O\MtDec.obj \ $O\Ppmd7.obj \ $O\Ppmd7Dec.obj \ $O\Ppmd7Enc.obj \ @@ -287,3 +290,4 @@ C_OBJS = \ !include "../../Aes.mak" !include "../../Crc.mak" !include "../../Crc64.mak" +!include "../../LzmaDec.mak" diff --git a/CPP/7zip/Bundles/Format7zF/Format7z.dsp b/CPP/7zip/Bundles/Format7zF/Format7z.dsp index 94368826..39ff535b 100644 --- a/CPP/7zip/Bundles/Format7zF/Format7z.dsp +++ b/CPP/7zip/Bundles/Format7zF/Format7z.dsp @@ -961,6 +961,24 @@ SOURCE=..\..\Compress\HuffmanDecoder.h # End Source File # Begin Source File +SOURCE=..\..\Compress\LzfseDecoder.cpp + +!IF "$(CFG)" == "7z - Win32 Release" + +# ADD CPP /O2 +# SUBTRACT CPP /YX /Yc /Yu + +!ELSEIF "$(CFG)" == "7z - Win32 Debug" + +!ENDIF + +# End Source File +# Begin Source File + +SOURCE=..\..\Compress\LzfseDecoder.h +# End Source File +# Begin Source File + SOURCE=..\..\Compress\LzhDecoder.cpp # End Source File # Begin Source File @@ -1787,6 +1805,26 @@ SOURCE=..\..\..\..\C\Lzma2Dec.h # End Source File # Begin Source File +SOURCE=..\..\..\..\C\Lzma2DecMt.c + +!IF "$(CFG)" == "7z - Win32 Release" + +# ADD CPP /O2 +# SUBTRACT CPP /YX /Yc /Yu + +!ELSEIF "$(CFG)" == "7z - Win32 Debug" + +# SUBTRACT CPP /YX /Yc /Yu + +!ENDIF + +# End Source File +# Begin Source File + +SOURCE=..\..\..\..\C\Lzma2DecMt.h +# End Source File +# Begin Source File + SOURCE=..\..\..\..\C\Lzma2Enc.c !IF "$(CFG)" == "7z - Win32 Release" @@ -1856,6 +1894,26 @@ SOURCE=..\..\..\..\C\MtCoder.h # End Source File # Begin Source File +SOURCE=..\..\..\..\C\MtDec.c + +!IF "$(CFG)" == "7z - Win32 Release" + +# ADD CPP /O2 +# SUBTRACT CPP /YX /Yc /Yu + +!ELSEIF "$(CFG)" == "7z - Win32 Debug" + +# SUBTRACT CPP /YX /Yc /Yu + +!ENDIF + +# End Source File +# Begin Source File + +SOURCE=..\..\..\..\C\MtDec.h +# End Source File +# Begin Source File + SOURCE=..\..\..\..\C\Ppmd.h # End Source File # Begin Source File diff --git a/CPP/7zip/Bundles/Format7zR/makefile b/CPP/7zip/Bundles/Format7zR/makefile index cd63c72d..6a9dfb91 100644 --- a/CPP/7zip/Bundles/Format7zR/makefile +++ b/CPP/7zip/Bundles/Format7zR/makefile @@ -102,12 +102,15 @@ C_OBJS = \ $O\LzFind.obj \ $O\LzFindMt.obj \ $O\Lzma2Dec.obj \ + $O\Lzma2DecMt.obj \ $O\Lzma2Enc.obj \ $O\LzmaDec.obj \ $O\LzmaEnc.obj \ $O\MtCoder.obj \ + $O\MtDec.obj \ $O\Threads.obj \ !include "../../Crc.mak" +!include "../../LzmaDec.mak" !include "../../7zip.mak" diff --git a/CPP/7zip/Bundles/SFXCon/SFXCon.dsp b/CPP/7zip/Bundles/SFXCon/SFXCon.dsp index f1e9a9ea..87753191 100644 --- a/CPP/7zip/Bundles/SFXCon/SFXCon.dsp +++ b/CPP/7zip/Bundles/SFXCon/SFXCon.dsp @@ -117,6 +117,10 @@ SOURCE=..\..\Archive\Common\CoderMixer2.h # End Source File # Begin Source File +SOURCE=..\..\Archive\Common\HandlerOut.h +# End Source File +# Begin Source File + SOURCE=..\..\Archive\Common\ItemNameUtils.cpp # End Source File # Begin Source File @@ -293,6 +297,10 @@ SOURCE=..\..\Compress\Lzma2Decoder.cpp # End Source File # Begin Source File +SOURCE=..\..\Compress\Lzma2Decoder.h +# End Source File +# Begin Source File + SOURCE=..\..\Compress\Lzma2Register.cpp # End Source File # Begin Source File @@ -411,6 +419,14 @@ SOURCE=..\..\..\Windows\Synchronization.cpp SOURCE=..\..\..\Windows\Synchronization.h # End Source File +# Begin Source File + +SOURCE=..\..\..\Windows\System.cpp +# End Source File +# Begin Source File + +SOURCE=..\..\..\Windows\System.h +# End Source File # End Group # Begin Group "Common" @@ -822,6 +838,15 @@ SOURCE=..\..\..\..\C\Lzma2Dec.h # End Source File # Begin Source File +SOURCE=..\..\..\..\C\Lzma2DecMt.c +# SUBTRACT CPP /YX /Yc /Yu +# End Source File +# Begin Source File + +SOURCE=..\..\..\..\C\Lzma2DecMt.h +# End Source File +# Begin Source File + SOURCE=..\..\..\..\C\LzmaDec.c # SUBTRACT CPP /YX /Yc /Yu # End Source File @@ -831,6 +856,15 @@ SOURCE=..\..\..\..\C\LzmaDec.h # End Source File # Begin Source File +SOURCE=..\..\..\..\C\MtDec.c +# SUBTRACT CPP /YX /Yc /Yu +# End Source File +# Begin Source File + +SOURCE=..\..\..\..\C\MtDec.h +# End Source File +# Begin Source File + SOURCE=..\..\..\..\C\Ppmd7.c # SUBTRACT CPP /YX /Yc /Yu # End Source File diff --git a/CPP/7zip/Bundles/SFXCon/makefile b/CPP/7zip/Bundles/SFXCon/makefile index 5476013b..fdf0f5cf 100644 --- a/CPP/7zip/Bundles/SFXCon/makefile +++ b/CPP/7zip/Bundles/SFXCon/makefile @@ -1,5 +1,7 @@ PROG = 7zCon.sfx MY_CONSOLE = 1 +MY_FIXED = 1 + CFLAGS = $(CFLAGS) \ -DEXTRACT_ONLY \ -DNO_READ_FROM_CODER \ @@ -118,7 +120,9 @@ C_OBJS = \ $O\Delta.obj \ $O\DllSecur.obj \ $O\Lzma2Dec.obj \ + $O\Lzma2DecMt.obj \ $O\LzmaDec.obj \ + $O\MtDec.obj \ $O\Ppmd7.obj \ $O\Ppmd7Dec.obj \ $O\Sha256.obj \ @@ -126,6 +130,7 @@ C_OBJS = \ !include "../../Aes.mak" !include "../../Crc.mak" +!include "../../LzmaDec.mak" COMPRESS_OBJS = $(COMPRESS_OBJS) \ $O\ZstdDecoder.obj \ diff --git a/CPP/7zip/Bundles/SFXSetup/SFXSetup.dsp b/CPP/7zip/Bundles/SFXSetup/SFXSetup.dsp index 4bf2fb83..b1d740e6 100644 --- a/CPP/7zip/Bundles/SFXSetup/SFXSetup.dsp +++ b/CPP/7zip/Bundles/SFXSetup/SFXSetup.dsp @@ -449,6 +449,14 @@ SOURCE=..\..\..\Windows\Synchronization.h # End Source File # Begin Source File +SOURCE=..\..\..\Windows\System.cpp +# End Source File +# Begin Source File + +SOURCE=..\..\..\Windows\System.h +# End Source File +# Begin Source File + SOURCE=..\..\..\Windows\Window.cpp # End Source File # Begin Source File @@ -469,6 +477,14 @@ SOURCE=..\..\Common\CreateCoder.h # End Source File # Begin Source File +SOURCE=..\..\Common\CWrappers.cpp +# End Source File +# Begin Source File + +SOURCE=..\..\Common\CWrappers.h +# End Source File +# Begin Source File + SOURCE=..\..\Common\FileStreams.cpp # End Source File # Begin Source File @@ -724,6 +740,15 @@ SOURCE=..\..\..\..\C\Lzma2Dec.h # End Source File # Begin Source File +SOURCE=..\..\..\..\C\Lzma2DecMt.c +# SUBTRACT CPP /YX /Yc /Yu +# End Source File +# Begin Source File + +SOURCE=..\..\..\..\C\Lzma2DecMt.h +# End Source File +# Begin Source File + SOURCE=..\..\..\..\C\LzmaDec.c # SUBTRACT CPP /YX /Yc /Yu # End Source File @@ -733,6 +758,15 @@ SOURCE=..\..\..\..\C\LzmaDec.h # End Source File # Begin Source File +SOURCE=..\..\..\..\C\MtDec.c +# SUBTRACT CPP /YX /Yc /Yu +# End Source File +# Begin Source File + +SOURCE=..\..\..\..\C\MtDec.h +# End Source File +# Begin Source File + SOURCE=..\..\..\..\C\Threads.c # SUBTRACT CPP /YX /Yc /Yu # End Source File diff --git a/CPP/7zip/Bundles/SFXSetup/makefile b/CPP/7zip/Bundles/SFXSetup/makefile index 2e6eed00..dd559990 100644 --- a/CPP/7zip/Bundles/SFXSetup/makefile +++ b/CPP/7zip/Bundles/SFXSetup/makefile @@ -1,4 +1,6 @@ PROG = 7zS.sfx +MY_FIXED = 1 + CFLAGS = $(CFLAGS) \ -DNO_REGISTRY \ -DEXTRACT_ONLY \ @@ -33,6 +35,7 @@ WIN_OBJS = \ $O\PropVariant.obj \ $O\ResourceString.obj \ $O\Synchronization.obj \ + $O\System.obj \ $O\Window.obj \ WIN_CTRL_OBJS = \ @@ -40,6 +43,7 @@ WIN_CTRL_OBJS = \ 7ZIP_COMMON_OBJS = \ $O\CreateCoder.obj \ + $O\CWrappers.obj \ $O\FileStreams.obj \ $O\InBuffer.obj \ $O\FilterCoder.obj \ @@ -102,8 +106,11 @@ C_OBJS = \ $O\Delta.obj \ $O\DllSecur.obj \ $O\Lzma2Dec.obj \ + $O\Lzma2DecMt.obj \ $O\LzmaDec.obj \ + $O\MtDec.obj \ $O\Threads.obj \ !include "../../Crc.mak" -!include "../../7zip.mak" +!include "../../LzmaDec.mak" +!include "../../7zip.mak" \ No newline at end of file diff --git a/CPP/7zip/Bundles/SFXWin/SFXWin.dsp b/CPP/7zip/Bundles/SFXWin/SFXWin.dsp index 9ee6c779..83ec9311 100644 --- a/CPP/7zip/Bundles/SFXWin/SFXWin.dsp +++ b/CPP/7zip/Bundles/SFXWin/SFXWin.dsp @@ -633,6 +633,14 @@ SOURCE=..\..\..\Windows\Synchronization.h # End Source File # Begin Source File +SOURCE=..\..\..\Windows\System.cpp +# End Source File +# Begin Source File + +SOURCE=..\..\..\Windows\System.h +# End Source File +# Begin Source File + SOURCE=..\..\..\Windows\Window.cpp # End Source File # Begin Source File @@ -906,6 +914,15 @@ SOURCE=..\..\..\..\C\Lzma2Dec.h # End Source File # Begin Source File +SOURCE=..\..\..\..\C\Lzma2DecMt.c +# SUBTRACT CPP /YX /Yc /Yu +# End Source File +# Begin Source File + +SOURCE=..\..\..\..\C\Lzma2DecMt.h +# End Source File +# Begin Source File + SOURCE=..\..\..\..\C\LzmaDec.c # SUBTRACT CPP /YX /Yc /Yu # End Source File @@ -915,6 +932,15 @@ SOURCE=..\..\..\..\C\LzmaDec.h # End Source File # Begin Source File +SOURCE=..\..\..\..\C\MtDec.c +# SUBTRACT CPP /YX /Yc /Yu +# End Source File +# Begin Source File + +SOURCE=..\..\..\..\C\MtDec.h +# End Source File +# Begin Source File + SOURCE=..\..\..\..\C\Ppmd7.c # SUBTRACT CPP /YX /Yc /Yu # End Source File diff --git a/CPP/7zip/Bundles/SFXWin/makefile b/CPP/7zip/Bundles/SFXWin/makefile index 61dcd692..ad64bbc5 100644 --- a/CPP/7zip/Bundles/SFXWin/makefile +++ b/CPP/7zip/Bundles/SFXWin/makefile @@ -1,4 +1,6 @@ PROG = 7z.sfx +MY_FIXED = 1 + CFLAGS = $(CFLAGS) \ -DNO_REGISTRY \ -DEXTRACT_ONLY \ @@ -42,6 +44,7 @@ WIN_OBJS = \ $O\Shell.obj \ $O\System.obj \ $O\Synchronization.obj \ + $O\System.obj \ $O\Window.obj \ WIN_CTRL_OBJS = \ @@ -136,7 +139,9 @@ C_OBJS = \ $O\Delta.obj \ $O\DllSecur.obj \ $O\Lzma2Dec.obj \ + $O\Lzma2DecMt.obj \ $O\LzmaDec.obj \ + $O\MtDec.obj \ $O\Ppmd7.obj \ $O\Ppmd7Dec.obj \ $O\Sha256.obj \ @@ -144,6 +149,7 @@ C_OBJS = \ !include "../../Aes.mak" !include "../../Crc.mak" +!include "../../LzmaDec.mak" COMPRESS_OBJS = $(COMPRESS_OBJS) \ $O\ZstdDecoder.obj \ diff --git a/CPP/7zip/Common/CWrappers.cpp b/CPP/7zip/Common/CWrappers.cpp index df914cd8..e726dadc 100644 --- a/CPP/7zip/Common/CWrappers.cpp +++ b/CPP/7zip/Common/CWrappers.cpp @@ -28,14 +28,23 @@ HRESULT SResToHRESULT(SRes res) throw() switch (res) { case SZ_OK: return S_OK; - case SZ_ERROR_DATA: return S_FALSE; - case SZ_ERROR_CRC: return S_FALSE; + + case SZ_ERROR_DATA: + case SZ_ERROR_CRC: + case SZ_ERROR_INPUT_EOF: + return S_FALSE; + case SZ_ERROR_MEM: return E_OUTOFMEMORY; case SZ_ERROR_PARAM: return E_INVALIDARG; case SZ_ERROR_PROGRESS: return E_ABORT; case SZ_ERROR_UNSUPPORTED: return E_NOTIMPL; - // case SZ_ERROR_THREAD: return E_FAIL; - // case SZ_ERROR_READ: return E_FAIL; + // case SZ_ERROR_OUTPUT_EOF: + // case SZ_ERROR_READ: + // case SZ_ERROR_WRITE: + // case SZ_ERROR_THREAD: + // case SZ_ERROR_ARCHIVE: + // case SZ_ERROR_NO_ARCHIVE: + // return E_FAIL; } if (res < 0) return res; diff --git a/CPP/7zip/Common/CreateCoder.cpp b/CPP/7zip/Common/CreateCoder.cpp index 53e7631e..50407657 100644 --- a/CPP/7zip/Common/CreateCoder.cpp +++ b/CPP/7zip/Common/CreateCoder.cpp @@ -148,20 +148,23 @@ HRESULT CExternalCodecs::Load() #endif -bool FindMethod( +int FindMethod_Index( DECL_EXTERNAL_CODECS_LOC_VARS const AString &name, - CMethodId &methodId, UInt32 &numStreams) + bool encode, + CMethodId &methodId, + UInt32 &numStreams) { unsigned i; for (i = 0; i < g_NumCodecs; i++) { const CCodecInfo &codec = *g_Codecs[i]; - if (StringsAreEqualNoCase_Ascii(name, codec.Name)) + if ((encode ? codec.CreateEncoder : codec.CreateDecoder) + && StringsAreEqualNoCase_Ascii(name, codec.Name)) { methodId = codec.Id; numStreams = codec.NumStreams; - return true; + return i; } } @@ -173,19 +176,51 @@ bool FindMethod( for (i = 0; i < __externalCodecs->Codecs.Size(); i++) { const CCodecInfoEx &codec = __externalCodecs->Codecs[i]; - if (StringsAreEqualNoCase_Ascii(name, codec.Name)) + if ((encode ? codec.EncoderIsAssigned : codec.DecoderIsAssigned) + && StringsAreEqualNoCase_Ascii(name, codec.Name)) { methodId = codec.Id; numStreams = codec.NumStreams; - return true; + return g_NumCodecs + i; } } #endif - return false; + return -1; } + +static int FindMethod_Index( + DECL_EXTERNAL_CODECS_LOC_VARS + CMethodId methodId, bool encode) +{ + unsigned i; + for (i = 0; i < g_NumCodecs; i++) + { + const CCodecInfo &codec = *g_Codecs[i]; + if (codec.Id == methodId && (encode ? codec.CreateEncoder : codec.CreateDecoder)) + return i; + } + + #ifdef EXTERNAL_CODECS + + CHECK_GLOBAL_CODECS + + if (__externalCodecs) + for (i = 0; i < __externalCodecs->Codecs.Size(); i++) + { + const CCodecInfoEx &codec = __externalCodecs->Codecs[i]; + if (codec.Id == methodId && (encode ? codec.EncoderIsAssigned : codec.DecoderIsAssigned)) + return g_NumCodecs + i; + } + + #endif + + return -1; +} + + bool FindMethod( DECL_EXTERNAL_CODECS_LOC_VARS CMethodId methodId, @@ -280,9 +315,11 @@ void GetHashMethods( #endif } -HRESULT CreateCoder( + + +HRESULT CreateCoder_Index( DECL_EXTERNAL_CODECS_LOC_VARS - CMethodId methodId, bool encode, + unsigned i, bool encode, CMyComPtr &filter, CCreatedCoder &cod) { @@ -290,11 +327,10 @@ HRESULT CreateCoder( cod.IsFilter = false; cod.NumStreams = 1; - unsigned i; - for (i = 0; i < g_NumCodecs; i++) + if (i < g_NumCodecs) { const CCodecInfo &codec = *g_Codecs[i]; - if (codec.Id == methodId) + // if (codec.Id == methodId) { if (encode) { @@ -325,11 +361,12 @@ HRESULT CreateCoder( if (__externalCodecs) { + i -= g_NumCodecs; cod.IsExternal = true; - for (i = 0; i < __externalCodecs->Codecs.Size(); i++) + if (i < __externalCodecs->Codecs.Size()) { const CCodecInfoEx &codec = __externalCodecs->Codecs[i]; - if (codec.Id == methodId) + // if (codec.Id == methodId) { if (encode) { @@ -371,13 +408,50 @@ HRESULT CreateCoder( return S_OK; } -HRESULT CreateCoder( + +HRESULT CreateCoder_Index( + DECL_EXTERNAL_CODECS_LOC_VARS + unsigned index, bool encode, + CCreatedCoder &cod) +{ + CMyComPtr filter; + HRESULT res = CreateCoder_Index( + EXTERNAL_CODECS_LOC_VARS + index, encode, + filter, cod); + + if (filter) + { + cod.IsFilter = true; + CFilterCoder *coderSpec = new CFilterCoder(encode); + cod.Coder = coderSpec; + coderSpec->Filter = filter; + } + + return res; +} + + +HRESULT CreateCoder_Id( + DECL_EXTERNAL_CODECS_LOC_VARS + CMethodId methodId, bool encode, + CMyComPtr &filter, + CCreatedCoder &cod) +{ + int index = FindMethod_Index(EXTERNAL_CODECS_LOC_VARS methodId, encode); + if (index < 0) + return S_OK; + return CreateCoder_Index(EXTERNAL_CODECS_LOC_VARS index, encode, filter, cod); +} + + +HRESULT CreateCoder_Id( DECL_EXTERNAL_CODECS_LOC_VARS CMethodId methodId, bool encode, CCreatedCoder &cod) { CMyComPtr filter; - HRESULT res = CreateCoder( + HRESULT res = CreateCoder_Id( EXTERNAL_CODECS_LOC_VARS methodId, encode, filter, cod); @@ -393,13 +467,14 @@ HRESULT CreateCoder( return res; } -HRESULT CreateCoder( + +HRESULT CreateCoder_Id( DECL_EXTERNAL_CODECS_LOC_VARS CMethodId methodId, bool encode, CMyComPtr &coder) { CCreatedCoder cod; - HRESULT res = CreateCoder( + HRESULT res = CreateCoder_Id( EXTERNAL_CODECS_LOC_VARS methodId, encode, cod); @@ -413,7 +488,7 @@ HRESULT CreateFilter( CMyComPtr &filter) { CCreatedCoder cod; - return CreateCoder( + return CreateCoder_Id( EXTERNAL_CODECS_LOC_VARS methodId, encode, filter, cod); diff --git a/CPP/7zip/Common/CreateCoder.h b/CPP/7zip/Common/CreateCoder.h index e0956e14..2105818f 100644 --- a/CPP/7zip/Common/CreateCoder.h +++ b/CPP/7zip/Common/CreateCoder.h @@ -116,13 +116,12 @@ extern CExternalCodecs g_ExternalCodecs; #endif - - - -bool FindMethod( +int FindMethod_Index( DECL_EXTERNAL_CODECS_LOC_VARS const AString &name, - CMethodId &methodId, UInt32 &numStreams); + bool encode, + CMethodId &methodId, + UInt32 &numStreams); bool FindMethod( DECL_EXTERNAL_CODECS_LOC_VARS @@ -152,18 +151,29 @@ struct CCreatedCoder }; -HRESULT CreateCoder( +HRESULT CreateCoder_Index( + DECL_EXTERNAL_CODECS_LOC_VARS + unsigned codecIndex, bool encode, + CMyComPtr &filter, + CCreatedCoder &cod); + +HRESULT CreateCoder_Index( + DECL_EXTERNAL_CODECS_LOC_VARS + unsigned index, bool encode, + CCreatedCoder &cod); + +HRESULT CreateCoder_Id( DECL_EXTERNAL_CODECS_LOC_VARS CMethodId methodId, bool encode, CMyComPtr &filter, CCreatedCoder &cod); -HRESULT CreateCoder( +HRESULT CreateCoder_Id( DECL_EXTERNAL_CODECS_LOC_VARS CMethodId methodId, bool encode, CCreatedCoder &cod); -HRESULT CreateCoder( +HRESULT CreateCoder_Id( DECL_EXTERNAL_CODECS_LOC_VARS CMethodId methodId, bool encode, CMyComPtr &coder); diff --git a/CPP/7zip/Common/InBuffer.cpp b/CPP/7zip/Common/InBuffer.cpp index 7765d1b7..826e98b1 100644 --- a/CPP/7zip/Common/InBuffer.cpp +++ b/CPP/7zip/Common/InBuffer.cpp @@ -97,6 +97,33 @@ Byte CInBufferBase::ReadByte_FromNewBlock() size_t CInBufferBase::ReadBytes(Byte *buf, size_t size) { + size_t num = 0; + for (;;) + { + const size_t rem = _bufLim - _buf; + if (size <= rem) + { + if (size != 0) + { + memcpy(buf, _buf, size); + _buf += size; + num += size; + } + return num; + } + if (rem != 0) + { + memcpy(buf, _buf, rem); + _buf += rem; + buf += rem; + num += rem; + size -= rem; + } + if (!ReadBlock()) + return num; + } + + /* if ((size_t)(_bufLim - _buf) >= size) { const Byte *src = _buf; @@ -113,6 +140,7 @@ size_t CInBufferBase::ReadBytes(Byte *buf, size_t size) buf[i] = *_buf++; } return size; + */ } size_t CInBufferBase::Skip(size_t size) diff --git a/CPP/7zip/Common/MethodProps.cpp b/CPP/7zip/Common/MethodProps.cpp index d4903809..2134462c 100644 --- a/CPP/7zip/Common/MethodProps.cpp +++ b/CPP/7zip/Common/MethodProps.cpp @@ -253,6 +253,9 @@ struct CNameToPropID const char *Name; }; + +// the following are related to NCoderPropID::EEnum values + static const CNameToPropID g_NameToPropID[] = { { VT_UI4, "" }, @@ -275,7 +278,8 @@ static const CNameToPropID g_NameToPropID[] = { VT_UI8, "expect" }, { VT_UI4, "b" }, { VT_UI4, "check" }, - { VT_BSTR, "filter" } + { VT_BSTR, "filter" }, + { VT_UI8, "memuse" } }; static int FindPropIdExact(const UString &name) @@ -293,6 +297,13 @@ static bool ConvertProperty(const PROPVARIANT &srcProp, VARTYPE varType, NCOM::C destProp = srcProp; return true; } + + if (varType == VT_UI8 && srcProp.vt == VT_UI4) + { + destProp = (UInt64)srcProp.ulVal; + return true; + } + if (varType == VT_BOOL) { bool res; diff --git a/CPP/7zip/Compress/BZip2Const.h b/CPP/7zip/Compress/BZip2Const.h index 33571ac2..b8bfd62a 100644 --- a/CPP/7zip/Compress/BZip2Const.h +++ b/CPP/7zip/Compress/BZip2Const.h @@ -50,6 +50,17 @@ const UInt32 kNumSelectorsMax = (2 + (kBlockSizeMax / kGroupSize)); const unsigned kRleModeRepSize = 4; +/* +The number of selectors stored in bzip2 block: +(numSelectors <= 18001) - must work with any decoder. +(numSelectors == 18002) - works with bzip2 1.0.6 decoder and all derived decoders. +(numSelectors > 18002) + 7-Zip decoder doesn't support it. + bzip2 1.0.6 decoder can overflow selector[18002] arrays. But there are another + arrays after selector arrays. So the compiled code works. + lbzip2 2.5 encoder can write up to (18001 + 7) selectors. +*/ + }} #endif diff --git a/CPP/7zip/Compress/BZip2Decoder.cpp b/CPP/7zip/Compress/BZip2Decoder.cpp index 21c87158..9f8f8b0e 100644 --- a/CPP/7zip/Compress/BZip2Decoder.cpp +++ b/CPP/7zip/Compress/BZip2Decoder.cpp @@ -411,9 +411,13 @@ SRes CBase::ReadBlock2() lens[state4] = (Byte)state3; state5 = 0; } + + // lbzip2 2.5 can produce dummy tree, where lens[i] = kMaxHuffmanLen + // BuildFull() returns error for such tree /* for (unsigned i = state4; i < kMaxAlphaSize; i++) lens[i] = 0; + if (!huffs[state2].Build(lens)) */ if (!huffs[state2].BuildFull(lens, state4)) return SZ_ERROR_DATA; @@ -474,7 +478,7 @@ SRes CBase::ReadBlock2() for (len = kNumTableBits + 1; val >= huff->_limits[len]; len++); /* if (len > kNumBitsMax) - return SZ_ERROR_DATA; + return SZ_ERROR_DATA; // that check is required, if NHuffman::Build() was used instead of BuildFull() */ if (_numBits < len) { diff --git a/CPP/7zip/Compress/DeflateEncoder.cpp b/CPP/7zip/Compress/DeflateEncoder.cpp index 3fb66d1a..009322e4 100644 --- a/CPP/7zip/Compress/DeflateEncoder.cpp +++ b/CPP/7zip/Compress/DeflateEncoder.cpp @@ -969,6 +969,10 @@ HRESULT CCoder::CodeReal(ISequentialInStream *inStream, ISequentialOutStream *ou } } while (Inline_MatchFinder_GetNumAvailableBytes(&_lzInWindow) != 0); + + if (_seqInStream.Res != S_OK) + return _seqInStream.Res; + if (_lzInWindow.result != SZ_OK) return SResToHRESULT(_lzInWindow.result); return m_OutStream.Flush(); diff --git a/CPP/7zip/Compress/LzOutWindow.h b/CPP/7zip/Compress/LzOutWindow.h index af578f91..eac963ba 100644 --- a/CPP/7zip/Compress/LzOutWindow.h +++ b/CPP/7zip/Compress/LzOutWindow.h @@ -56,6 +56,39 @@ public: if (pos == _limitPos) FlushWithCheck(); } + + void PutBytes(const Byte *data, UInt32 size) + { + if (size == 0) + return; + UInt32 pos = _pos; + Byte *buf = _buf; + buf[pos++] = *data++; + size--; + for (;;) + { + UInt32 limitPos = _limitPos; + UInt32 rem = limitPos - pos; + if (rem == 0) + { + _pos = pos; + FlushWithCheck(); + pos = _pos; + continue; + } + + if (size == 0) + break; + + if (rem > size) + rem = size; + size -= rem; + do + buf[pos++] = *data++; + while (--rem); + } + _pos = pos; + } Byte GetByte(UInt32 distance) const { diff --git a/CPP/7zip/Compress/LzfseDecoder.cpp b/CPP/7zip/Compress/LzfseDecoder.cpp new file mode 100644 index 00000000..548ca995 --- /dev/null +++ b/CPP/7zip/Compress/LzfseDecoder.cpp @@ -0,0 +1,925 @@ +// LzfseDecoder.cpp + +/* +This code implements LZFSE data decompressing. +The code from "LZFSE compression library" was used. + +2018 : Igor Pavlov : BSD 3-clause License : the code in this file +2015-2017 : Apple Inc : BSD 3-clause License : original "LZFSE compression library" code + +The code in the "LZFSE compression library" is licensed under the "BSD 3-clause License": +---- +Copyright (c) 2015-2016, Apple Inc. All rights reserved. + +Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: + +1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. + +2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer + in the documentation and/or other materials provided with the distribution. + +3. Neither the name of the copyright holder(s) nor the names of any contributors may be used to endorse or promote products derived + from this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE +COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) +HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +---- +*/ + +#include "StdAfx.h" + +// #define SHOW_DEBUG_INFO + +#ifdef SHOW_DEBUG_INFO +#include +#endif + +#ifdef SHOW_DEBUG_INFO +#define PRF(x) x +#else +#define PRF(x) +#endif + +#include "../../../C/CpuArch.h" + +#include "LzfseDecoder.h" + +namespace NCompress { +namespace NLzfse { + +static const Byte kSignature_LZFSE_V1 = 0x31; // '1' +static const Byte kSignature_LZFSE_V2 = 0x32; // '2' + + +HRESULT CDecoder::GetUInt32(UInt32 &val) +{ + Byte b[4]; + for (unsigned i = 0; i < 4; i++) + if (!m_InStream.ReadByte(b[i])) + return S_FALSE; + val = GetUi32(b); + return S_OK; +} + + + +HRESULT CDecoder::DecodeUncompressed(UInt32 unpackSize) +{ + PRF(printf("\nUncompressed %7u\n", unpackSize)); + + const unsigned kBufSize = 1 << 8; + Byte buf[kBufSize]; + for (;;) + { + if (unpackSize == 0) + return S_OK; + UInt32 cur = unpackSize; + if (cur > kBufSize) + cur = kBufSize; + UInt32 cur2 = (UInt32)m_InStream.ReadBytes(buf, cur); + m_OutWindowStream.PutBytes(buf, cur2); + if (cur != cur2) + return S_FALSE; + } +} + + + +HRESULT CDecoder::DecodeLzvn(UInt32 unpackSize) +{ + UInt32 packSize; + RINOK(GetUInt32(packSize)); + + PRF(printf("\nLZVN %7u %7u", unpackSize, packSize)); + + UInt32 D = 0; + + for (;;) + { + if (packSize == 0) + return S_FALSE; + Byte b; + if (!m_InStream.ReadByte(b)) + return S_FALSE; + packSize--; + + UInt32 M; + UInt32 L; + + if (b >= 0xE0) + { + /* + large L - 11100000 LLLLLLLL + small L - 1110LLLL + + large Rep - 11110000 MMMMMMMM + small Rep - 1111MMMM + */ + + M = b & 0xF; + if (M == 0) + { + if (packSize == 0) + return S_FALSE; + Byte b1; + if (!m_InStream.ReadByte(b1)) + return S_FALSE; + packSize--; + M = (UInt32)b1 + 16; + } + L = 0; + if ((b & 0x10) == 0) + { + // Literals only + L = M; + M = 0; + } + } + + // ERROR codes + else if ((b & 0xF0) == 0x70) // 0111xxxx + return S_FALSE; + else if ((b & 0xF0) == 0xD0) // 1101xxxx + return S_FALSE; + + else + { + if ((b & 0xE0) == 0xA0) + { + // medium - 101LLMMM DDDDDDMM DDDDDDDD + if (packSize < 2) + return S_FALSE; + Byte b1; + if (!m_InStream.ReadByte(b1)) + return S_FALSE; + packSize--; + + Byte b2; + if (!m_InStream.ReadByte(b2)) + return S_FALSE; + packSize--; + L = (((UInt32)b >> 3) & 3); + M = (((UInt32)b & 7) << 2) + (b1 & 3); + D = ((UInt32)b1 >> 2) + ((UInt32)b2 << 6); + } + else + { + L = (UInt32)b >> 6; + M = ((UInt32)b >> 3) & 7; + if ((b & 0x7) == 6) + { + // REP - LLMMM110 + if (L == 0) + { + // spec + if (M == 0) + break; // EOS + if (M <= 2) + continue; // NOP + return S_FALSE; // UNDEFINED + } + } + else + { + if (packSize == 0) + return S_FALSE; + Byte b1; + if (!m_InStream.ReadByte(b1)) + return S_FALSE; + packSize--; + + // large - LLMMM111 DDDDDDDD DDDDDDDD + // small - LLMMMDDD DDDDDDDD + D = ((UInt32)b & 7); + if (D == 7) + { + if (packSize == 0) + return S_FALSE; + Byte b2; + if (!m_InStream.ReadByte(b2)) + return S_FALSE; + packSize--; + D = b2; + } + D = (D << 8) + b1; + } + } + + M += 3; + } + { + for (unsigned i = 0; i < L; i++) + { + if (packSize == 0 || unpackSize == 0) + return S_FALSE; + Byte b1; + if (!m_InStream.ReadByte(b1)) + return S_FALSE; + packSize--; + m_OutWindowStream.PutByte(b1); + unpackSize--; + } + } + + if (M != 0) + { + if (unpackSize == 0 || D == 0) + return S_FALSE; + unsigned cur = M; + if (cur > unpackSize) + cur = (unsigned)unpackSize; + if (!m_OutWindowStream.CopyBlock(D - 1, cur)) + return S_FALSE; + unpackSize -= cur; + if (cur != M) + return S_FALSE; + } + } + + if (unpackSize != 0) + return S_FALSE; + + // LZVN encoder writes 7 additional zero bytes + if (packSize != 7) + return S_FALSE; + do + { + Byte b; + if (!m_InStream.ReadByte(b)) + return S_FALSE; + packSize--; + if (b != 0) + return S_FALSE; + } + while (packSize != 0); + + return S_OK; +} + + + +// ---------- LZFSE ---------- + +#define MATCHES_PER_BLOCK 10000 +#define LITERALS_PER_BLOCK (4 * MATCHES_PER_BLOCK) + +#define NUM_L_SYMBOLS 20 +#define NUM_M_SYMBOLS 20 +#define NUM_D_SYMBOLS 64 +#define NUM_LIT_SYMBOLS 256 + +#define NUM_SYMBOLS ( \ + NUM_L_SYMBOLS + \ + NUM_M_SYMBOLS + \ + NUM_D_SYMBOLS + \ + NUM_LIT_SYMBOLS) + +#define NUM_L_STATES (1 << 6) +#define NUM_M_STATES (1 << 6) +#define NUM_D_STATES (1 << 8) +#define NUM_LIT_STATES (1 << 10) + + +typedef UInt32 CFseState; + + +static UInt32 SumFreqs(const UInt16 *freqs, unsigned num) +{ + UInt32 sum = 0; + for (unsigned i = 0; i < num; i++) + sum += (UInt32)freqs[i]; + return sum; +} + + +static MY_FORCE_INLINE unsigned CountZeroBits(UInt32 val, UInt32 mask) +{ + for (unsigned i = 0;;) + { + if (val & mask) + return i; + i++; + mask >>= 1; + } +} + + +static MY_FORCE_INLINE void InitLitTable(const UInt16 *freqs, UInt32 *table) +{ + for (unsigned i = 0; i < NUM_LIT_SYMBOLS; i++) + { + unsigned f = freqs[i]; + if (f == 0) + continue; + + // 0 < f <= numStates + // 0 <= k <= numStatesLog + // numStates <= (f<> k) - f; + + /* + CEntry + { + Byte k; + Byte symbol; + UInt16 delta; + }; + */ + + UInt32 e = ((UInt32)i << 8) + k; + k += 16; + UInt32 d = e + ((UInt32)f << k) - ((UInt32)NUM_LIT_STATES << 16); + UInt32 step = (UInt32)1 << k; + + unsigned j = 0; + do + { + *table++ = d; + d += step; + } + while (++j < j0); + + e--; + step >>= 1; + + for (j = j0; j < f; j++) + { + *table++ = e; + e += step; + } + } +} + + +typedef struct +{ + Byte totalBits; + Byte extraBits; + UInt16 delta; + UInt32 vbase; +} CExtraEntry; + + +static void InitExtraDecoderTable(unsigned numStates, + unsigned numSymbols, + const UInt16 *freqs, + const Byte *vbits, + CExtraEntry *table) +{ + UInt32 vbase = 0; + + for (unsigned i = 0; i < numSymbols; i++) + { + unsigned f = freqs[i]; + unsigned extraBits = vbits[i]; + + if (f != 0) + { + unsigned k = CountZeroBits(f, numStates); + unsigned j0 = ((2 * numStates) >> k) - f; + + unsigned j = 0; + do + { + CExtraEntry *e = table++; + e->totalBits = (Byte)(k + extraBits); + e->extraBits = (Byte)extraBits; + e->delta = (UInt16)(((f + j) << k) - numStates); + e->vbase = vbase; + } + while (++j < j0); + + f -= j0; + k--; + + for (j = 0; j < f; j++) + { + CExtraEntry *e = table++; + e->totalBits = (Byte)(k + extraBits); + e->extraBits = (Byte)extraBits; + e->delta = (UInt16)(j << k); + e->vbase = vbase; + } + } + + vbase += ((UInt32)1 << extraBits); + } +} + + +static const Byte k_L_extra[NUM_L_SYMBOLS] = +{ + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 3, 5, 8 +}; + +static const Byte k_M_extra[NUM_M_SYMBOLS] = +{ + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 5, 8, 11 +}; + +static const Byte k_D_extra[NUM_D_SYMBOLS] = +{ + 0, 0, 0, 0, 1, 1, 1, 1, 2, 2, 2, 2, 3, 3, 3, 3, + 4, 4, 4, 4, 5, 5, 5, 5, 6, 6, 6, 6, 7, 7, 7, 7, + 8, 8, 8, 8, 9, 9, 9, 9, 10, 10, 10, 10, 11, 11, 11, 11, + 12, 12, 12, 12, 13, 13, 13, 13, 14, 14, 14, 14, 15, 15, 15, 15 +}; + + + +// ---------- CBitStream ---------- + +typedef struct +{ + UInt32 accum; + unsigned numBits; // [0, 31] - Number of valid bits in (accum), other bits are 0 +} CBitStream; + + +static MY_FORCE_INLINE int FseInStream_Init(CBitStream *s, + int n, // [-7, 0], (-n == number_of_unused_bits) in last byte + const Byte **pbuf) +{ + *pbuf -= 4; + s->accum = GetUi32(*pbuf); + if (n) + { + s->numBits = n + 32; + if ((s->accum >> s->numBits) != 0) + return -1; // ERROR, encoder should have zeroed the upper bits + } + else + { + *pbuf += 1; + s->accum >>= 8; + s->numBits = 24; + } + return 0; // OK +} + + +// 0 <= numBits < 32 +#define mask31(x, numBits) ((x) & (((UInt32)1 << (numBits)) - 1)) + +#define FseInStream_FLUSH \ + { unsigned nbits = (31 - in.numBits) & -8; \ + if (nbits) { \ + buf -= (nbits >> 3); \ + if (buf < buf_check) return S_FALSE; \ + UInt32 v = GetUi32(buf); \ + in.accum = (in.accum << nbits) | mask31(v, nbits); \ + in.numBits += nbits; }} + + + +static MY_FORCE_INLINE UInt32 BitStream_Pull(CBitStream *s, unsigned numBits) +{ + s->numBits -= numBits; + UInt32 v = s->accum >> s->numBits; + s->accum = mask31(s->accum, s->numBits); + return v; +} + + +#define DECODE_LIT(dest, pstate) { \ + UInt32 e = lit_decoder[pstate]; \ + pstate = (CFseState)((e >> 16) + BitStream_Pull(&in, e & 0xff)); \ + dest = (Byte)(e >> 8); } + + +static MY_FORCE_INLINE UInt32 FseDecodeExtra(CFseState *pstate, + const CExtraEntry *table, + CBitStream *s) +{ + const CExtraEntry *e = &table[*pstate]; + UInt32 v = BitStream_Pull(s, e->totalBits); + unsigned extraBits = e->extraBits; + *pstate = (CFseState)(e->delta + (v >> extraBits)); + return e->vbase + mask31(v, extraBits); +} + + +#define freqs_L (freqs) +#define freqs_M (freqs_L + NUM_L_SYMBOLS) +#define freqs_D (freqs_M + NUM_M_SYMBOLS) +#define freqs_LIT (freqs_D + NUM_D_SYMBOLS) + +#define GET_BITS_64(v, offset, num, dest) dest = (UInt32) ((v >> (offset)) & ((1 << (num)) - 1)); +#define GET_BITS_32(v, offset, num, dest) dest = (CFseState)((v >> (offset)) & ((1 << (num)) - 1)); + + +HRESULT CDecoder::DecodeLzfse(UInt32 unpackSize, Byte version) +{ + PRF(printf("\nLZFSE-%d %7u", version - '0', unpackSize)); + + UInt32 numLiterals; + UInt32 litPayloadSize; + Int32 literal_bits; + + UInt32 lit_state_0; + UInt32 lit_state_1; + UInt32 lit_state_2; + UInt32 lit_state_3; + + UInt32 numMatches; + UInt32 lmdPayloadSize; + Int32 lmd_bits; + + CFseState l_state; + CFseState m_state; + CFseState d_state; + + UInt16 freqs[NUM_SYMBOLS]; + + if (version == kSignature_LZFSE_V1) + { + return E_NOTIMPL; + // we need examples to test LZFSE-V1 code + /* + const unsigned k_v1_SubHeaderSize = 7 * 4 + 7 * 2; + const unsigned k_v1_HeaderSize = k_v1_SubHeaderSize + NUM_SYMBOLS * 2; + _buffer.AllocAtLeast(k_v1_HeaderSize); + if (m_InStream.ReadBytes(_buffer, k_v1_HeaderSize) != k_v1_HeaderSize) + return S_FALSE; + + const Byte *buf = _buffer; + #define GET_32(offs, dest) dest = GetUi32(buf + offs) + #define GET_16(offs, dest) dest = GetUi16(buf + offs) + + UInt32 payload_bytes; + GET_32(0, payload_bytes); + GET_32(4, numLiterals); + GET_32(8, numMatches); + GET_32(12, litPayloadSize); + GET_32(16, lmdPayloadSize); + if (litPayloadSize > (1 << 20) || lmdPayloadSize > (1 << 20)) + return S_FALSE; + GET_32(20, literal_bits); + if (literal_bits < -7 || literal_bits > 0) + return S_FALSE; + + GET_16(24, lit_state_0); + GET_16(26, lit_state_1); + GET_16(28, lit_state_2); + GET_16(30, lit_state_3); + + GET_32(32, lmd_bits); + if (lmd_bits < -7 || lmd_bits > 0) + return S_FALSE; + + GET_16(36, l_state); + GET_16(38, m_state); + GET_16(40, d_state); + + for (unsigned i = 0; i < NUM_SYMBOLS; i++) + freqs[i] = GetUi16(buf + k_v1_SubHeaderSize + i * 2); + */ + } + else + { + UInt32 headerSize; + { + const unsigned kPreHeaderSize = 4 * 2; // signature and upackSize + const unsigned kHeaderSize = 8 * 3; + Byte temp[kHeaderSize]; + if (m_InStream.ReadBytes(temp, kHeaderSize) != kHeaderSize) + return S_FALSE; + + UInt64 v; + + v = GetUi64(temp); + GET_BITS_64(v, 0, 20, numLiterals); + GET_BITS_64(v, 20, 20, litPayloadSize); + GET_BITS_64(v, 40, 20, numMatches); + GET_BITS_64(v, 60, 3 + 1, literal_bits); // (NumberOfUsedBits - 1) + literal_bits -= 7; // (-NumberOfUnusedBits) + if (literal_bits > 0) + return S_FALSE; + // GET_BITS_64(v, 63, 1, unused); + + v = GetUi64(temp + 8); + GET_BITS_64(v, 0, 10, lit_state_0); + GET_BITS_64(v, 10, 10, lit_state_1); + GET_BITS_64(v, 20, 10, lit_state_2); + GET_BITS_64(v, 30, 10, lit_state_3); + GET_BITS_64(v, 40, 20, lmdPayloadSize); + GET_BITS_64(v, 60, 3 + 1, lmd_bits); + lmd_bits -= 7; + if (lmd_bits > 0) + return S_FALSE; + // GET_BITS_64(v, 63, 1, unused) + + UInt32 v32 = GetUi32(temp + 20); + // (total header size in bytes; this does not + // correspond to a field in the uncompressed header version, + // but is required; we wouldn't know the size of the + // compresssed header otherwise. + GET_BITS_32(v32, 0, 10, l_state); + GET_BITS_32(v32, 10, 10, m_state); + GET_BITS_32(v32, 20, 10 + 2, d_state); + // GET_BITS_64(v, 62, 2, unused); + + headerSize = GetUi32(temp + 16); + if (headerSize <= kPreHeaderSize + kHeaderSize) + return S_FALSE; + headerSize -= kPreHeaderSize + kHeaderSize; + } + + // no freqs case is not allowed ? + // memset(freqs, 0, sizeof(freqs)); + // if (headerSize != 0) + { + static const Byte numBitsTable[32] = + { + 2, 3, 2, 5, 2, 3, 2, 8, 2, 3, 2, 5, 2, 3, 2, 14, + 2, 3, 2, 5, 2, 3, 2, 8, 2, 3, 2, 5, 2, 3, 2, 14 + }; + + static const Byte valueTable[32] = + { + 0, 2, 1, 4, 0, 3, 1, 8, 0, 2, 1, 5, 0, 3, 1, 24, + 0, 2, 1, 6, 0, 3, 1, 8, 0, 2, 1, 7, 0, 3, 1, 24 + }; + + UInt32 accum = 0; + unsigned numBits = 0; + + for (unsigned i = 0; i < NUM_SYMBOLS; i++) + { + while (numBits <= 14 && headerSize != 0) + { + Byte b; + if (!m_InStream.ReadByte(b)) + return S_FALSE; + accum |= (UInt32)b << numBits; + numBits += 8; + headerSize--; + } + + unsigned b = (unsigned)accum & 31; + unsigned n = numBitsTable[b]; + if (numBits < n) + return S_FALSE; + numBits -= n; + UInt32 f = valueTable[b]; + if (n >= 8) + f += ((accum >> 4) & (0x3ff >> (14 - n))); + accum >>= n; + freqs[i] = (UInt16)f; + } + + if (numBits >= 8 || headerSize != 0) + return S_FALSE; + } + } + + PRF(printf(" Literals=%6u Matches=%6u", numLiterals, numMatches)); + + if (numLiterals > LITERALS_PER_BLOCK + || (numLiterals & 3) != 0 + || numMatches > MATCHES_PER_BLOCK + || lit_state_0 >= NUM_LIT_STATES + || lit_state_1 >= NUM_LIT_STATES + || lit_state_2 >= NUM_LIT_STATES + || lit_state_3 >= NUM_LIT_STATES + || l_state >= NUM_L_STATES + || m_state >= NUM_M_STATES + || d_state >= NUM_D_STATES) + return S_FALSE; + + // only full table is allowed ? + if ( SumFreqs(freqs_L, NUM_L_SYMBOLS) != NUM_L_STATES + || SumFreqs(freqs_M, NUM_M_SYMBOLS) != NUM_M_STATES + || SumFreqs(freqs_D, NUM_D_SYMBOLS) != NUM_D_STATES + || SumFreqs(freqs_LIT, NUM_LIT_SYMBOLS) != NUM_LIT_STATES) + return S_FALSE; + + + const unsigned kPad = 16; + + // ---------- Decode literals ---------- + + { + _literals.AllocAtLeast(LITERALS_PER_BLOCK + 16); + _buffer.AllocAtLeast(kPad + litPayloadSize); + memset(_buffer, 0, kPad); + + if (m_InStream.ReadBytes(_buffer + kPad, litPayloadSize) != litPayloadSize) + return S_FALSE; + + UInt32 lit_decoder[NUM_LIT_STATES]; + InitLitTable(freqs_LIT, lit_decoder); + + const Byte *buf_start = _buffer + kPad; + const Byte *buf_check = buf_start - 4; + const Byte *buf = buf_start + litPayloadSize; + CBitStream in; + if (FseInStream_Init(&in, literal_bits, &buf) != 0) + return S_FALSE; + + Byte *lit = _literals; + const Byte *lit_limit = lit + numLiterals; + for (; lit < lit_limit; lit += 4) + { + FseInStream_FLUSH + DECODE_LIT (lit[0], lit_state_0); + DECODE_LIT (lit[1], lit_state_1); + FseInStream_FLUSH + DECODE_LIT (lit[2], lit_state_2); + DECODE_LIT (lit[3], lit_state_3); + } + + if ((buf_start - buf) * 8 != (int)in.numBits) + return S_FALSE; + } + + + // ---------- Decode LMD ---------- + + _buffer.AllocAtLeast(kPad + lmdPayloadSize); + memset(_buffer, 0, kPad); + if (m_InStream.ReadBytes(_buffer + kPad, lmdPayloadSize) != lmdPayloadSize) + return S_FALSE; + + CExtraEntry l_decoder[NUM_L_STATES]; + CExtraEntry m_decoder[NUM_M_STATES]; + CExtraEntry d_decoder[NUM_D_STATES]; + + InitExtraDecoderTable(NUM_L_STATES, NUM_L_SYMBOLS, freqs_L, k_L_extra, l_decoder); + InitExtraDecoderTable(NUM_M_STATES, NUM_M_SYMBOLS, freqs_M, k_M_extra, m_decoder); + InitExtraDecoderTable(NUM_D_STATES, NUM_D_SYMBOLS, freqs_D, k_D_extra, d_decoder); + + const Byte *buf_start = _buffer + kPad; + const Byte *buf_check = buf_start - 4; + const Byte *buf = buf_start + lmdPayloadSize; + CBitStream in; + if (FseInStream_Init(&in, lmd_bits, &buf)) + return S_FALSE; + + const Byte *lit = _literals; + const Byte *lit_limit = lit + numLiterals; + + UInt32 D = 0; + + for (;;) + { + if (numMatches == 0) + break; + numMatches--; + + FseInStream_FLUSH + + unsigned L = (unsigned)FseDecodeExtra(&l_state, l_decoder, &in); + + FseInStream_FLUSH + + unsigned M = (unsigned)FseDecodeExtra(&m_state, m_decoder, &in); + + FseInStream_FLUSH + + { + UInt32 new_D = FseDecodeExtra(&d_state, d_decoder, &in); + if (new_D) + D = new_D; + } + + if (L != 0) + { + if (L > (size_t)(lit_limit - lit)) + return S_FALSE; + unsigned cur = L; + if (cur > unpackSize) + cur = (unsigned)unpackSize; + m_OutWindowStream.PutBytes(lit, cur); + unpackSize -= cur; + lit += cur; + if (cur != L) + return S_FALSE; + } + + if (M != 0) + { + if (unpackSize == 0 || D == 0) + return S_FALSE; + unsigned cur = M; + if (cur > unpackSize) + cur = (unsigned)unpackSize; + if (!m_OutWindowStream.CopyBlock(D - 1, cur)) + return S_FALSE; + unpackSize -= cur; + if (cur != M) + return S_FALSE; + } + } + + if (unpackSize != 0) + return S_FALSE; + + // LZFSE encoder writes 8 additional zero bytes before LMD payload + // We test it: + if ((buf - buf_start) * 8 + in.numBits != 64) + return S_FALSE; + if (GetUi64(buf_start) != 0) + return S_FALSE; + + return S_OK; +} + + +STDMETHODIMP CDecoder::CodeReal(ISequentialInStream *inStream, ISequentialOutStream *outStream, + const UInt64 *inSize, const UInt64 *outSize, ICompressProgressInfo *progress) +{ + PRF(printf("\n\nLzfseDecoder %7u %7u\n", (unsigned)*outSize, (unsigned)*inSize)); + + const UInt32 kLzfseDictSize = 1 << 18; + if (!m_OutWindowStream.Create(kLzfseDictSize)) + return E_OUTOFMEMORY; + if (!m_InStream.Create(1 << 18)) + return E_OUTOFMEMORY; + + m_OutWindowStream.SetStream(outStream); + m_OutWindowStream.Init(false); + m_InStream.SetStream(inStream); + m_InStream.Init(); + + CCoderReleaser coderReleaser(this); + + UInt64 prevOut = 0; + UInt64 prevIn = 0; + + for (;;) + { + const UInt64 pos = m_OutWindowStream.GetProcessedSize(); + const UInt64 packPos = m_InStream.GetProcessedSize(); + + if (progress && ((pos - prevOut) >= (1 << 22) || (packPos - prevIn) >= (1 << 22))) + { + RINOK(progress->SetRatioInfo(&packPos, &pos)); + prevIn = packPos; + prevOut = pos; + } + + const UInt64 rem = *outSize - pos; + UInt32 v; + RINOK(GetUInt32(v)) + if ((v & 0xFFFFFF) != 0x787662) // bvx + return S_FALSE; + v >>= 24; + + if (v == 0x24) // '$', end of stream + break; + + UInt32 unpackSize; + RINOK(GetUInt32(unpackSize)); + + UInt32 cur = unpackSize; + if (cur > rem) + cur = (UInt32)rem; + + unpackSize -= cur; + + HRESULT res; + if (v == kSignature_LZFSE_V1 || v == kSignature_LZFSE_V2) + res = DecodeLzfse(cur, (Byte)v); + else if (v == 0x6E) // 'n' + res = DecodeLzvn(cur); + else if (v == 0x2D) // '-' + res = DecodeUncompressed(cur); + else + return E_NOTIMPL; + + if (res != S_OK) + return res; + + if (unpackSize != 0) + return S_FALSE; + } + + coderReleaser.NeedFlush = false; + HRESULT res = m_OutWindowStream.Flush(); + if (res == S_OK) + if (*inSize != m_InStream.GetProcessedSize() + || *outSize != m_OutWindowStream.GetProcessedSize()) + res = S_FALSE; + return res; +} + + +STDMETHODIMP CDecoder::Code(ISequentialInStream *inStream, ISequentialOutStream *outStream, + const UInt64 *inSize, const UInt64 *outSize, ICompressProgressInfo *progress) +{ + try { return CodeReal(inStream, outStream, inSize, outSize, progress); } + catch(const CInBufferException &e) { return e.ErrorCode; } + catch(const CLzOutWindowException &e) { return e.ErrorCode; } + catch(...) { return E_OUTOFMEMORY; } + // catch(...) { return S_FALSE; } +} + +}} diff --git a/CPP/7zip/Compress/LzfseDecoder.h b/CPP/7zip/Compress/LzfseDecoder.h new file mode 100644 index 00000000..9d2b354f --- /dev/null +++ b/CPP/7zip/Compress/LzfseDecoder.h @@ -0,0 +1,58 @@ +// LzfseDecoder.h + +#ifndef __LZFSE_DECODER_H +#define __LZFSE_DECODER_H + +#include "../../Common/MyBuffer.h" +#include "../../Common/MyCom.h" + +#include "../ICoder.h" + +#include "../Common/InBuffer.h" + +#include "LzOutWindow.h" + +namespace NCompress { +namespace NLzfse { + +class CDecoder: + public ICompressCoder, + public CMyUnknownImp +{ + CLzOutWindow m_OutWindowStream; + CInBuffer m_InStream; + CByteBuffer _literals; + CByteBuffer _buffer; + + class CCoderReleaser + { + CDecoder *m_Coder; + public: + bool NeedFlush; + CCoderReleaser(CDecoder *coder): m_Coder(coder), NeedFlush(true) {} + ~CCoderReleaser() + { + if (NeedFlush) + m_Coder->m_OutWindowStream.Flush(); + } + }; + friend class CCoderReleaser; + + HRESULT GetUInt32(UInt32 &val); + + HRESULT DecodeUncompressed(UInt32 unpackSize); + HRESULT DecodeLzvn(UInt32 unpackSize); + HRESULT DecodeLzfse(UInt32 unpackSize, Byte version); + + STDMETHOD(CodeReal)(ISequentialInStream *inStream, ISequentialOutStream *outStream, + const UInt64 *inSize, const UInt64 *outSize, ICompressProgressInfo *progress); +public: + MY_UNKNOWN_IMP + + STDMETHOD(Code)(ISequentialInStream *inStream, ISequentialOutStream *outStream, const UInt64 *inSize, + const UInt64 *outSize, ICompressProgressInfo *progress); +}; + +}} + +#endif diff --git a/CPP/7zip/Compress/Lzma2Decoder.cpp b/CPP/7zip/Compress/Lzma2Decoder.cpp index 746743eb..bb631c5c 100644 --- a/CPP/7zip/Compress/Lzma2Decoder.cpp +++ b/CPP/7zip/Compress/Lzma2Decoder.cpp @@ -2,81 +2,48 @@ #include "StdAfx.h" +// #include + #include "../../../C/Alloc.h" +// #include "../../../C/CpuTicks.h" #include "../Common/StreamUtils.h" #include "Lzma2Decoder.h" -static HRESULT SResToHRESULT(SRes res) -{ - switch (res) - { - case SZ_OK: return S_OK; - case SZ_ERROR_MEM: return E_OUTOFMEMORY; - case SZ_ERROR_PARAM: return E_INVALIDARG; - case SZ_ERROR_UNSUPPORTED: return E_NOTIMPL; - case SZ_ERROR_DATA: return S_FALSE; - } - return E_FAIL; -} - namespace NCompress { namespace NLzma2 { CDecoder::CDecoder(): - _inBuf(NULL), - _finishMode(false), - _outSizeDefined(false), - _outStep(1 << 22), - _inBufSize(0), - _inBufSizeNew(1 << 20) -{ - Lzma2Dec_Construct(&_state); -} + _dec(NULL) + , _inProcessed(0) + , _prop(0xFF) + , _finishMode(false) + , _inBufSize(1 << 20) + , _outStep(1 << 20) + #ifndef _7ZIP_ST + , _tryMt(1) + , _numThreads(1) + , _memUsage((UInt64)(sizeof(size_t)) << 28) + #endif +{} CDecoder::~CDecoder() { - Lzma2Dec_Free(&_state, &g_Alloc); - MidFree(_inBuf); + if (_dec) + Lzma2DecMt_Destroy(_dec); } -STDMETHODIMP CDecoder::SetInBufSize(UInt32 , UInt32 size) { _inBufSizeNew = size; return S_OK; } +STDMETHODIMP CDecoder::SetInBufSize(UInt32 , UInt32 size) { _inBufSize = size; return S_OK; } STDMETHODIMP CDecoder::SetOutBufSize(UInt32 , UInt32 size) { _outStep = size; return S_OK; } STDMETHODIMP CDecoder::SetDecoderProperties2(const Byte *prop, UInt32 size) { if (size != 1) return E_NOTIMPL; - - RINOK(SResToHRESULT(Lzma2Dec_Allocate(&_state, prop[0], &g_Alloc))); - - if (!_inBuf || _inBufSize != _inBufSizeNew) - { - MidFree(_inBuf); - _inBufSize = 0; - _inBuf = (Byte *)MidAlloc(_inBufSizeNew); - if (!_inBuf) - return E_OUTOFMEMORY; - _inBufSize = _inBufSizeNew; - } - - return S_OK; -} - - -STDMETHODIMP CDecoder::SetOutStreamSize(const UInt64 *outSize) -{ - _outSizeDefined = (outSize != NULL); - _outSize = 0; - if (_outSizeDefined) - _outSize = *outSize; - _inPos = _inLim = 0; - _inProcessed = 0; - _outProcessed = 0; - - Lzma2Dec_Init(&_state); - + if (prop[0] > 40) + return E_NOTIMPL; + _prop = prop[0]; return S_OK; } @@ -88,6 +55,137 @@ STDMETHODIMP CDecoder::SetFinishMode(UInt32 finishMode) } + +#ifndef _7ZIP_ST + +static UInt64 Get_ExpectedBlockSize_From_Dict(UInt32 dictSize) +{ + const UInt32 kMinSize = (UInt32)1 << 20; + const UInt32 kMaxSize = (UInt32)1 << 28; + UInt64 blockSize = (UInt64)dictSize << 2; + if (blockSize < kMinSize) blockSize = kMinSize; + if (blockSize > kMaxSize) blockSize = kMaxSize; + if (blockSize < dictSize) blockSize = dictSize; + blockSize += (kMinSize - 1); + blockSize &= ~(UInt64)(kMinSize - 1); + return blockSize; +} + +#define LZMA2_DIC_SIZE_FROM_PROP_FULL(p) ((p) == 40 ? 0xFFFFFFFF : (((UInt32)2 | ((p) & 1)) << ((p) / 2 + 11))) + +#endif + +#define RET_IF_WRAP_ERROR_CONFIRMED(wrapRes, sRes, sResErrorCode) \ + if (wrapRes != S_OK && sRes == sResErrorCode) return wrapRes; + +#define RET_IF_WRAP_ERROR(wrapRes, sRes, sResErrorCode) \ + if (wrapRes != S_OK /* && (sRes == SZ_OK || sRes == sResErrorCode) */) return wrapRes; + +STDMETHODIMP CDecoder::Code(ISequentialInStream *inStream, ISequentialOutStream *outStream, + const UInt64 *inSize, const UInt64 *outSize, ICompressProgressInfo *progress) +{ + _inProcessed = 0; + + if (!_dec) + { + _dec = Lzma2DecMt_Create( + // &g_AlignedAlloc, + &g_Alloc, + &g_MidAlloc); + if (!_dec) + return E_OUTOFMEMORY; + } + + CLzma2DecMtProps props; + Lzma2DecMtProps_Init(&props); + + props.inBufSize_ST = _inBufSize; + props.outStep_ST = _outStep; + + #ifndef _7ZIP_ST + { + props.numThreads = 1; + UInt32 numThreads = _numThreads; + + if (_tryMt && numThreads >= 1) + { + UInt64 useLimit = _memUsage; + UInt32 dictSize = LZMA2_DIC_SIZE_FROM_PROP_FULL(_prop); + UInt64 expectedBlockSize64 = Get_ExpectedBlockSize_From_Dict(dictSize); + size_t expectedBlockSize = (size_t)expectedBlockSize64; + size_t inBlockMax = expectedBlockSize + expectedBlockSize / 16; + if (expectedBlockSize == expectedBlockSize64 && inBlockMax >= expectedBlockSize) + { + props.outBlockMax = expectedBlockSize; + props.inBlockMax = inBlockMax; + const size_t kOverheadSize = props.inBufSize_MT + (1 << 16); + UInt64 okThreads = useLimit / (props.outBlockMax + props.inBlockMax + kOverheadSize); + if (numThreads > okThreads) + numThreads = (UInt32)okThreads; + if (numThreads == 0) + numThreads = 1; + props.numThreads = numThreads; + } + } + } + #endif + + CSeqInStreamWrap inWrap; + CSeqOutStreamWrap outWrap; + CCompressProgressWrap progressWrap; + + inWrap.Init(inStream); + outWrap.Init(outStream); + progressWrap.Init(progress); + + SRes res; + + UInt64 inProcessed = 0; + int isMT = False; + + #ifndef _7ZIP_ST + isMT = _tryMt; + #endif + + // UInt64 cpuTicks = GetCpuTicks(); + + res = Lzma2DecMt_Decode(_dec, _prop, &props, + &outWrap.vt, outSize, _finishMode, + &inWrap.vt, + &inProcessed, + &isMT, + progress ? &progressWrap.vt : NULL); + + /* + cpuTicks = GetCpuTicks() - cpuTicks; + printf("\n ticks = %10I64u\n", cpuTicks / 1000000); + */ + + + #ifndef _7ZIP_ST + /* we reset _tryMt, only if p->props.numThreads was changed */ + if (props.numThreads > 1) + _tryMt = isMT; + #endif + + _inProcessed = inProcessed; + + RET_IF_WRAP_ERROR(progressWrap.Res, res, SZ_ERROR_PROGRESS) + RET_IF_WRAP_ERROR(outWrap.Res, res, SZ_ERROR_WRITE) + RET_IF_WRAP_ERROR_CONFIRMED(inWrap.Res, res, SZ_ERROR_READ) + + if (res == SZ_OK && _finishMode) + { + if (inSize && *inSize != inProcessed) + res = SZ_ERROR_DATA; + if (outSize && *outSize != outWrap.Processed) + res = SZ_ERROR_DATA; + } + + return SResToHRESULT(res); +} + + STDMETHODIMP CDecoder::GetInStreamProcessedSize(UInt64 *value) { *value = _inProcessed; @@ -95,109 +193,51 @@ STDMETHODIMP CDecoder::GetInStreamProcessedSize(UInt64 *value) } -STDMETHODIMP CDecoder::Code(ISequentialInStream *inStream, ISequentialOutStream *outStream, - const UInt64 *inSize, const UInt64 *outSize, ICompressProgressInfo *progress) +#ifndef _7ZIP_ST + +STDMETHODIMP CDecoder::SetNumberOfThreads(UInt32 numThreads) { - if (!_inBuf) - return S_FALSE; - - SetOutStreamSize(outSize); - - SizeT wrPos = _state.decoder.dicPos; - HRESULT readRes = S_OK; - - for (;;) - { - if (_inPos == _inLim && readRes == S_OK) - { - _inPos = _inLim = 0; - readRes = inStream->Read(_inBuf, _inBufSize, &_inLim); - } - - const SizeT dicPos = _state.decoder.dicPos; - SizeT size; - { - SizeT next = _state.decoder.dicBufSize; - if (next - wrPos > _outStep) - next = wrPos + _outStep; - size = next - dicPos; - } - - ELzmaFinishMode finishMode = LZMA_FINISH_ANY; - if (_outSizeDefined) - { - const UInt64 rem = _outSize - _outProcessed; - if (size >= rem) - { - size = (SizeT)rem; - if (_finishMode) - finishMode = LZMA_FINISH_END; - } - } - - SizeT inProcessed = _inLim - _inPos; - ELzmaStatus status; - - SRes res = Lzma2Dec_DecodeToDic(&_state, dicPos + size, _inBuf + _inPos, &inProcessed, finishMode, &status); - - - _inPos += (UInt32)inProcessed; - _inProcessed += inProcessed; - const SizeT outProcessed = _state.decoder.dicPos - dicPos; - _outProcessed += outProcessed; - - - bool outFinished = (_outSizeDefined && _outProcessed >= _outSize); - - bool needStop = (res != 0 - || (inProcessed == 0 && outProcessed == 0) - || status == LZMA_STATUS_FINISHED_WITH_MARK - || (!_finishMode && outFinished)); - - if (needStop || outProcessed >= size) - { - HRESULT res2 = WriteStream(outStream, _state.decoder.dic + wrPos, _state.decoder.dicPos - wrPos); - - if (_state.decoder.dicPos == _state.decoder.dicBufSize) - _state.decoder.dicPos = 0; - wrPos = _state.decoder.dicPos; - - RINOK(res2); - - if (needStop) - { - if (res != 0) - return S_FALSE; - - if (status == LZMA_STATUS_FINISHED_WITH_MARK) - { - if (_finishMode) - { - if (inSize && *inSize != _inProcessed) - return S_FALSE; - if (_outSizeDefined && _outSize != _outProcessed) - return S_FALSE; - } - return readRes; - } - - if (!_finishMode && outFinished) - return readRes; - - return S_FALSE; - } - } - - if (progress) - { - RINOK(progress->SetRatioInfo(&_inProcessed, &_outProcessed)); - } - } + _numThreads = numThreads; + return S_OK; } +STDMETHODIMP CDecoder::SetMemLimit(UInt64 memUsage) +{ + _memUsage = memUsage; + return S_OK; +} + +#endif + #ifndef NO_READ_FROM_CODER +STDMETHODIMP CDecoder::SetOutStreamSize(const UInt64 *outSize) +{ + CLzma2DecMtProps props; + Lzma2DecMtProps_Init(&props); + props.inBufSize_ST = _inBufSize; + props.outStep_ST = _outStep; + + _inProcessed = 0; + + if (!_dec) + { + _dec = Lzma2DecMt_Create(&g_AlignedAlloc, &g_MidAlloc); + if (!_dec) + return E_OUTOFMEMORY; + } + + _inWrap.Init(_inStream); + + SRes res = Lzma2DecMt_Init(_dec, _prop, &props, outSize, _finishMode, &_inWrap.vt); + + if (res != SZ_OK) + return SResToHRESULT(res); + return S_OK; +} + + STDMETHODIMP CDecoder::SetInStream(ISequentialInStream *inStream) { _inStream = inStream; return S_OK; } STDMETHODIMP CDecoder::ReleaseInStream() { _inStream.Release(); return S_OK; } @@ -207,62 +247,17 @@ STDMETHODIMP CDecoder::Read(void *data, UInt32 size, UInt32 *processedSize) if (processedSize) *processedSize = 0; - ELzmaFinishMode finishMode = LZMA_FINISH_ANY; - if (_outSizeDefined) - { - const UInt64 rem = _outSize - _outProcessed; - if (size >= rem) - { - size = (UInt32)rem; - if (_finishMode) - finishMode = LZMA_FINISH_END; - } - } + size_t size2 = size; + UInt64 inProcessed = 0; - HRESULT readRes = S_OK; + SRes res = Lzma2DecMt_Read(_dec, (Byte *)data, &size2, &inProcessed); - for (;;) - { - if (_inPos == _inLim && readRes == S_OK) - { - _inPos = _inLim = 0; - readRes = _inStream->Read(_inBuf, _inBufSize, &_inLim); - } - - SizeT inProcessed = _inLim - _inPos; - SizeT outProcessed = size; - ELzmaStatus status; - - SRes res = Lzma2Dec_DecodeToBuf(&_state, (Byte *)data, &outProcessed, - _inBuf + _inPos, &inProcessed, finishMode, &status); - - - _inPos += (UInt32)inProcessed; - _inProcessed += inProcessed; - _outProcessed += outProcessed; - size -= (UInt32)outProcessed; - data = (Byte *)data + outProcessed; - if (processedSize) - *processedSize += (UInt32)outProcessed; - - if (res != 0) - return S_FALSE; - - /* - if (status == LZMA_STATUS_FINISHED_WITH_MARK) - return readRes; - - if (size == 0 && status != LZMA_STATUS_NEEDS_MORE_INPUT) - { - if (_finishMode && _outSizeDefined && _outProcessed >= _outSize) - return S_FALSE; - return readRes; - } - */ - - if (inProcessed == 0 && outProcessed == 0) - return readRes; - } + _inProcessed += inProcessed; + if (processedSize) + *processedSize = (UInt32)size2; + if (res != SZ_OK) + return SResToHRESULT(res); + return S_OK; } #endif diff --git a/CPP/7zip/Compress/Lzma2Decoder.h b/CPP/7zip/Compress/Lzma2Decoder.h index d271043a..b56488e0 100644 --- a/CPP/7zip/Compress/Lzma2Decoder.h +++ b/CPP/7zip/Compress/Lzma2Decoder.h @@ -3,10 +3,9 @@ #ifndef __LZMA2_DECODER_H #define __LZMA2_DECODER_H -#include "../../../C/Lzma2Dec.h" +#include "../../../C/Lzma2DecMt.h" -#include "../../Common/MyCom.h" -#include "../ICoder.h" +#include "../Common/CWrappers.h" namespace NCompress { namespace NLzma2 { @@ -17,28 +16,26 @@ class CDecoder: public ICompressSetFinishMode, public ICompressGetInStreamProcessedSize, public ICompressSetBufSize, + #ifndef NO_READ_FROM_CODER public ICompressSetInStream, public ICompressSetOutStreamSize, public ISequentialInStream, #endif + + #ifndef _7ZIP_ST + public ICompressSetCoderMt, + public ICompressSetMemLimit, + #endif + public CMyUnknownImp { - Byte *_inBuf; - UInt32 _inPos; - UInt32 _inLim; - - bool _finishMode; - bool _outSizeDefined; - UInt64 _outSize; + CLzma2DecMtHandle _dec; UInt64 _inProcessed; - UInt64 _outProcessed; - - UInt32 _outStep; + Byte _prop; + int _finishMode; UInt32 _inBufSize; - UInt32 _inBufSizeNew; - - CLzma2Dec _state; + UInt32 _outStep; public: MY_QUERYINTERFACE_BEGIN2(ICompressCoder) @@ -46,11 +43,18 @@ public: MY_QUERYINTERFACE_ENTRY(ICompressSetFinishMode) MY_QUERYINTERFACE_ENTRY(ICompressGetInStreamProcessedSize) MY_QUERYINTERFACE_ENTRY(ICompressSetBufSize) + #ifndef NO_READ_FROM_CODER MY_QUERYINTERFACE_ENTRY(ICompressSetInStream) MY_QUERYINTERFACE_ENTRY(ICompressSetOutStreamSize) MY_QUERYINTERFACE_ENTRY(ISequentialInStream) #endif + + #ifndef _7ZIP_ST + MY_QUERYINTERFACE_ENTRY(ICompressSetCoderMt) + MY_QUERYINTERFACE_ENTRY(ICompressSetMemLimit) + #endif + MY_QUERYINTERFACE_END MY_ADDREF_RELEASE @@ -59,20 +63,28 @@ public: STDMETHOD(SetDecoderProperties2)(const Byte *data, UInt32 size); STDMETHOD(SetFinishMode)(UInt32 finishMode); STDMETHOD(GetInStreamProcessedSize)(UInt64 *value); - STDMETHOD(SetOutStreamSize)(const UInt64 *outSize); STDMETHOD(SetInBufSize)(UInt32 streamIndex, UInt32 size); STDMETHOD(SetOutBufSize)(UInt32 streamIndex, UInt32 size); - #ifndef NO_READ_FROM_CODER + #ifndef _7ZIP_ST +private: + int _tryMt; + UInt32 _numThreads; + UInt64 _memUsage; +public: + STDMETHOD(SetNumberOfThreads)(UInt32 numThreads); + STDMETHOD(SetMemLimit)(UInt64 memUsage); + #endif + #ifndef NO_READ_FROM_CODER private: CMyComPtr _inStream; + CSeqInStreamWrap _inWrap; public: - + STDMETHOD(SetOutStreamSize)(const UInt64 *outSize); STDMETHOD(SetInStream)(ISequentialInStream *inStream); STDMETHOD(ReleaseInStream)(); STDMETHOD(Read)(void *data, UInt32 size, UInt32 *processedSize); - #endif CDecoder(); diff --git a/CPP/7zip/Compress/Lzma2Encoder.cpp b/CPP/7zip/Compress/Lzma2Encoder.cpp index a16f9c13..18f7d029 100644 --- a/CPP/7zip/Compress/Lzma2Encoder.cpp +++ b/CPP/7zip/Compress/Lzma2Encoder.cpp @@ -22,7 +22,7 @@ namespace NLzma2 { CEncoder::CEncoder() { _encoder = NULL; - _encoder = Lzma2Enc_Create(&g_Alloc, &g_BigAlloc); + _encoder = Lzma2Enc_Create(&g_AlignedAlloc, &g_BigAlloc); if (!_encoder) throw 1; } diff --git a/CPP/7zip/Compress/LzmaDecoder.cpp b/CPP/7zip/Compress/LzmaDecoder.cpp index 3df0e6de..b6a8d3fb 100644 --- a/CPP/7zip/Compress/LzmaDecoder.cpp +++ b/CPP/7zip/Compress/LzmaDecoder.cpp @@ -30,19 +30,24 @@ CDecoder::CDecoder(): FinishStream(false), _propsWereSet(false), _outSizeDefined(false), - _outStep(1 << 22), + _outStep(1 << 20), _inBufSize(0), _inBufSizeNew(1 << 20) { _inProcessed = 0; _inPos = _inLim = 0; + /* + AlignOffsetAlloc_CreateVTable(&_alloc); + _alloc.numAlignBits = 7; + _alloc.offset = 0; + */ LzmaDec_Construct(&_state); } CDecoder::~CDecoder() { - LzmaDec_Free(&_state, &g_Alloc); + LzmaDec_Free(&_state, &g_AlignedAlloc); // &_alloc.vt MyFree(_inBuf); } @@ -66,7 +71,7 @@ HRESULT CDecoder::CreateInputBuffer() STDMETHODIMP CDecoder::SetDecoderProperties2(const Byte *prop, UInt32 size) { - RINOK(SResToHRESULT(LzmaDec_Allocate(&_state, prop, size, &g_Alloc))); + RINOK(SResToHRESULT(LzmaDec_Allocate(&_state, prop, size, &g_AlignedAlloc))) // &_alloc.vt _propsWereSet = true; return CreateInputBuffer(); } diff --git a/CPP/7zip/Compress/LzmaDecoder.h b/CPP/7zip/Compress/LzmaDecoder.h index 886b604c..08b7c1bc 100644 --- a/CPP/7zip/Compress/LzmaDecoder.h +++ b/CPP/7zip/Compress/LzmaDecoder.h @@ -3,6 +3,7 @@ #ifndef __LZMA_DECODER_H #define __LZMA_DECODER_H +// #include "../../../C/Alloc.h" #include "../../../C/LzmaDec.h" #include "../../Common/MyCom.h" @@ -28,7 +29,6 @@ class CDecoder: UInt32 _inPos; UInt32 _inLim; - CLzmaDec _state; ELzmaStatus _lzmaStatus; public: @@ -45,6 +45,10 @@ private: UInt32 _inBufSize; UInt32 _inBufSizeNew; + // CAlignOffsetAlloc _alloc; + + CLzmaDec _state; + HRESULT CreateInputBuffer(); HRESULT CodeSpec(ISequentialInStream *inStream, ISequentialOutStream *outStream, ICompressProgressInfo *progress); void SetOutStreamSizeResume(const UInt64 *outSize); diff --git a/CPP/7zip/Compress/LzmaEncoder.cpp b/CPP/7zip/Compress/LzmaEncoder.cpp index 6ce89bae..3151c3d2 100644 --- a/CPP/7zip/Compress/LzmaEncoder.cpp +++ b/CPP/7zip/Compress/LzmaEncoder.cpp @@ -15,7 +15,7 @@ namespace NLzma { CEncoder::CEncoder() { _encoder = NULL; - _encoder = LzmaEnc_Create(&g_Alloc); + _encoder = LzmaEnc_Create(&g_AlignedAlloc); if (!_encoder) throw 1; } @@ -23,7 +23,7 @@ CEncoder::CEncoder() CEncoder::~CEncoder() { if (_encoder) - LzmaEnc_Destroy(_encoder, &g_Alloc, &g_BigAlloc); + LzmaEnc_Destroy(_encoder, &g_AlignedAlloc, &g_BigAlloc); } static inline wchar_t GetUpperChar(wchar_t c) @@ -168,7 +168,7 @@ STDMETHODIMP CEncoder::Code(ISequentialInStream *inStream, ISequentialOutStream progressWrap.Init(progress); SRes res = LzmaEnc_Encode(_encoder, &outWrap.vt, &inWrap.vt, - progress ? &progressWrap.vt : NULL, &g_Alloc, &g_BigAlloc); + progress ? &progressWrap.vt : NULL, &g_AlignedAlloc, &g_BigAlloc); _inputProcessed = inWrap.Processed; diff --git a/CPP/7zip/Compress/Rar1Decoder.cpp b/CPP/7zip/Compress/Rar1Decoder.cpp index aca6bf8b..02de0704 100644 --- a/CPP/7zip/Compress/Rar1Decoder.cpp +++ b/CPP/7zip/Compress/Rar1Decoder.cpp @@ -9,77 +9,85 @@ namespace NCompress { namespace NRar1 { -static const UInt32 PosL1[] = {0,0,0,2,3,5,7,11,16,20,24,32,32, 256}; -static const UInt32 PosL2[] = {0,0,0,0,5,7,9,13,18,22,26,34,36, 256}; -static const UInt32 PosHf0[] = {0,0,0,0,0,8,16,24,33,33,33,33,33, 257}; -static const UInt32 PosHf1[] = {0,0,0,0,0,0,4,44,60,76,80,80,127, 257}; -static const UInt32 PosHf2[] = {0,0,0,0,0,0,2,7,53,117,233, 257,0}; -static const UInt32 PosHf3[] = {0,0,0,0,0,0,0,2,16,218,251, 257,0}; -static const UInt32 PosHf4[] = {0,0,0,0,0,0,0,0,0,255, 257,0,0}; +static const unsigned kNumBits = 12; + +static const Byte kShortLen1[16 * 3] = +{ + 0,0xa0,0xd0,0xe0,0xf0,0xf8,0xfc,0xfe,0xff,0xc0,0x80,0x90,0x98,0x9c,0xb0,0, + 1,3,4,4,5,6,7,8,8,4,4,5,6,6,0,0, + 1,4,4,4,5,6,7,8,8,4,4,5,6,6,4,0 +}; + +static const Byte kShortLen2[16 * 3] = +{ + 0,0x40,0x60,0xa0,0xd0,0xe0,0xf0,0xf8,0xfc,0xc0,0x80,0x90,0x98,0x9c,0xb0,0, + 2,3,3,3,4,4,5,6,6,4,4,5,6,6,0,0, + 2,3,3,4,4,4,5,6,6,4,4,5,6,6,4,0 +}; + +static const Byte PosL1[kNumBits + 1] = { 0,0,2,1,2,2,4,5,4,4,8,0,224 }; +static const Byte PosL2[kNumBits + 1] = { 0,0,0,5,2,2,4,5,4,4,8,2,220 }; + +static const Byte PosHf0[kNumBits + 1] = { 0,0,0,0,8,8,8,9,0,0,0,0,224 }; +static const Byte PosHf1[kNumBits + 1] = { 0,0,0,0,0,4,40,16,16,4,0,47,130 }; +static const Byte PosHf2[kNumBits + 1] = { 0,0,0,0,0,2,5,46,64,116,24,0,0 }; +static const Byte PosHf3[kNumBits + 1] = { 0,0,0,0,0,0,2,14,202,33,6,0,0 }; +static const Byte PosHf4[kNumBits + 1] = { 0,0,0,0,0,0,0,0,255,2,0,0,0 }; static const UInt32 kHistorySize = (1 << 16); -/* -class CCoderReleaser -{ - CDecoder *m_Coder; -public: - CCoderReleaser(CDecoder *coder): m_Coder(coder) {} - ~CCoderReleaser() { m_Coder->ReleaseStreams(); } -}; -*/ +CDecoder::CDecoder(): + _isSolid(false), + _solidAllowed(false) + { } -CDecoder::CDecoder(): m_IsSolid(false) { } - -void CDecoder::InitStructures() -{ - for (int i = 0; i < kNumRepDists; i++) - m_RepDists[i] = 0; - m_RepDistPtr = 0; - LastLength = 0; - LastDist = 0; -} - -UInt32 CDecoder::ReadBits(int numBits) { return m_InBitStream.ReadBits(numBits); } +UInt32 CDecoder::ReadBits(unsigned numBits) { return m_InBitStream.ReadBits(numBits); } HRESULT CDecoder::CopyBlock(UInt32 distance, UInt32 len) { if (len == 0) return S_FALSE; + if (m_UnpackSize < len) + return S_FALSE; m_UnpackSize -= len; return m_OutWindowStream.CopyBlock(distance, len) ? S_OK : S_FALSE; } -UInt32 CDecoder::DecodeNum(const UInt32 *posTab) +UInt32 CDecoder::DecodeNum(const Byte *numTab) { - UInt32 startPos = 2; - UInt32 num = m_InBitStream.GetValue(12); + /* + { + // we can check that tables are correct + UInt32 sum = 0; + for (unsigned i = 0; i <= kNumBits; i++) + sum += ((UInt32)numTab[i] << (kNumBits - i)); + if (sum != (1 << kNumBits)) + throw 111; + } + */ + + UInt32 val = m_InBitStream.GetValue(kNumBits); + UInt32 sum = 0; + unsigned i = 2; + for (;;) { - UInt32 cur = (posTab[(size_t)startPos + 1] - posTab[startPos]) << (12 - startPos); - if (num < cur) + UInt32 num = numTab[i]; + UInt32 cur = num << (kNumBits - i); + if (val < cur) break; - startPos++; - num -= cur; + i++; + val -= cur; + sum += num; } - m_InBitStream.MovePos(startPos); - return((num >> (12 - startPos)) + posTab[startPos]); + m_InBitStream.MovePos(i); + return ((val >> (kNumBits - i)) + sum); } -static const Byte kShortLen1 [] = {1,3,4,4,5,6,7,8,8,4,4,5,6,6 }; -static const Byte kShortLen1a[] = {1,4,4,4,5,6,7,8,8,4,4,5,6,6,4 }; -static const Byte kShortLen2 [] = {2,3,3,3,4,4,5,6,6,4,4,5,6,6 }; -static const Byte kShortLen2a[] = {2,3,3,4,4,4,5,6,6,4,4,5,6,6,4 }; -static const UInt32 kShortXor1[] = {0,0xa0,0xd0,0xe0,0xf0,0xf8,0xfc,0xfe,0xff,0xc0,0x80,0x90,0x98,0x9c,0xb0}; -static const UInt32 kShortXor2[] = {0,0x40,0x60,0xa0,0xd0,0xe0,0xf0,0xf8,0xfc,0xc0,0x80,0x90,0x98,0x9c,0xb0}; HRESULT CDecoder::ShortLZ() { - UInt32 len, saveLen, dist; - int distancePlace; - const Byte *kShortLen; - const UInt32 *kShortXor; NumHuf = 0; if (LCount == 2) @@ -91,20 +99,14 @@ HRESULT CDecoder::ShortLZ() UInt32 bitField = m_InBitStream.GetValue(8); - if (AvrLn1 < 37) + UInt32 len, dist; { - kShortLen = Buf60 ? kShortLen1a : kShortLen1; - kShortXor = kShortXor1; + const Byte *xors = (AvrLn1 < 37) ? kShortLen1 : kShortLen2; + const Byte *lens = xors + 16 + Buf60; + for (len = 0; ((bitField ^ xors[len]) >> (8 - lens[len])) != 0; len++); + m_InBitStream.MovePos(lens[len]); } - else - { - kShortLen = Buf60 ? kShortLen2a : kShortLen2; - kShortXor = kShortXor2; - } - - for (len = 0; ((bitField ^ kShortXor[len]) & (~(0xff >> kShortLen[len]))) != 0; len++); - m_InBitStream.MovePos(kShortLen[len]); - + if (len >= 9) { if (len == 9) @@ -112,9 +114,11 @@ HRESULT CDecoder::ShortLZ() LCount++; return CopyBlock(LastDist, LastLength); } + + LCount = 0; + if (len == 14) { - LCount = 0; len = DecodeNum(PosL2) + 5; dist = 0x8000 + ReadBits(15) - 1; LastLength = len; @@ -122,19 +126,22 @@ HRESULT CDecoder::ShortLZ() return CopyBlock(dist, len); } - LCount = 0; - saveLen = len; + UInt32 saveLen = len; dist = m_RepDists[(m_RepDistPtr - (len - 9)) & 3]; - len = DecodeNum(PosL1) + 2; - if (len == 0x101 && saveLen == 10) + + len = DecodeNum(PosL1); + + if (len == 0xff && saveLen == 10) { - Buf60 ^= 1; + Buf60 ^= 16; return S_OK; } if (dist >= 256) + { len++; - if (dist >= MaxDist3 - 1) - len++; + if (dist >= MaxDist3 - 1) + len++; + } } else { @@ -142,21 +149,23 @@ HRESULT CDecoder::ShortLZ() AvrLn1 += len; AvrLn1 -= AvrLn1 >> 4; - distancePlace = DecodeNum(PosHf2) & 0xff; - dist = ChSetA[(unsigned)distancePlace]; - if (--distancePlace != -1) + unsigned distancePlace = DecodeNum(PosHf2) & 0xff; + + dist = ChSetA[distancePlace]; + + if (distancePlace != 0) { PlaceA[dist]--; - UInt32 lastDistance = ChSetA[(unsigned)distancePlace]; + UInt32 lastDistance = ChSetA[(size_t)distancePlace - 1]; PlaceA[lastDistance]++; - ChSetA[(size_t)(unsigned)distancePlace + 1] = lastDistance; - ChSetA[(unsigned)distancePlace] = dist; + ChSetA[distancePlace] = lastDistance; + ChSetA[(size_t)distancePlace - 1] = dist; } - len += 2; } m_RepDists[m_RepDistPtr++] = dist; m_RepDistPtr &= 3; + len += 2; LastLength = len; LastDist = dist; return CopyBlock(dist, len); @@ -177,12 +186,10 @@ HRESULT CDecoder::LongLZ() Nlzb = 0x90; Nhfb >>= 1; } - oldAvr2=AvrLn2; + oldAvr2 = AvrLn2; - if (AvrLn2 >= 122) - len = DecodeNum(PosL2); - else if (AvrLn2 >= 64) - len = DecodeNum(PosL1); + if (AvrLn2 >= 64) + len = DecodeNum(AvrLn2 < 122 ? PosL1 : PosL2); else { UInt32 bitField = m_InBitStream.GetValue(16); @@ -193,8 +200,8 @@ HRESULT CDecoder::LongLZ() } else { - for (len = 0; ((bitField << len) & 0x8000) == 0; len++) - ; + for (len = 0; ((bitField << len) & 0x8000) == 0; len++); + m_InBitStream.MovePos(len + 1); } } @@ -202,24 +209,26 @@ HRESULT CDecoder::LongLZ() AvrLn2 += len; AvrLn2 -= AvrLn2 >> 5; - if (AvrPlcB > 0x28ff) - distancePlace = DecodeNum(PosHf2); - else if (AvrPlcB > 0x6ff) - distancePlace = DecodeNum(PosHf1); - else - distancePlace = DecodeNum(PosHf0); + { + const Byte *tab; + if (AvrPlcB >= 0x2900) tab = PosHf2; + else if (AvrPlcB >= 0x0700) tab = PosHf1; + else tab = PosHf0; + distancePlace = DecodeNum(tab); // [0, 256] + } AvrPlcB += distancePlace; AvrPlcB -= AvrPlcB >> 8; + + distancePlace &= 0xff; for (;;) { - dist = ChSetB[distancePlace & 0xff]; + dist = ChSetB[distancePlace]; newDistancePlace = NToPlB[dist++ & 0xff]++; - if (!(dist & 0xff)) - CorrHuff(ChSetB,NToPlB); - else + if (dist & 0xff) break; + CorrHuff(ChSetB,NToPlB); } ChSetB[distancePlace] = ChSetB[newDistancePlace]; @@ -235,9 +244,8 @@ HRESULT CDecoder::LongLZ() AvrLn3++; AvrLn3 -= AvrLn3 >> 8; } - else - if (AvrLn3 > 0) - AvrLn3--; + else if (AvrLn3 > 0) + AvrLn3--; len += 3; @@ -265,57 +273,62 @@ HRESULT CDecoder::HuffDecode() UInt32 curByte, newBytePlace; UInt32 len; UInt32 dist; - int bytePlace; - - if (AvrPlc > 0x75ff) bytePlace = DecodeNum(PosHf4); - else if (AvrPlc > 0x5dff) bytePlace = DecodeNum(PosHf3); - else if (AvrPlc > 0x35ff) bytePlace = DecodeNum(PosHf2); - else if (AvrPlc > 0x0dff) bytePlace = DecodeNum(PosHf1); - else bytePlace = DecodeNum(PosHf0); + unsigned bytePlace; + { + const Byte *tab; + + if (AvrPlc >= 0x7600) tab = PosHf4; + else if (AvrPlc >= 0x5e00) tab = PosHf3; + else if (AvrPlc >= 0x3600) tab = PosHf2; + else if (AvrPlc >= 0x0e00) tab = PosHf1; + else tab = PosHf0; + + bytePlace = DecodeNum(tab); // [0, 256] + } if (StMode) { - if (--bytePlace == -1) + if (bytePlace == 0) { if (ReadBits(1)) { - NumHuf = StMode = 0; + NumHuf = 0; + StMode = false; return S_OK; } - else - { - len = (ReadBits(1)) ? 4 : 3; - dist = DecodeNum(PosHf2); - dist = (dist << 5) | ReadBits(5); - return CopyBlock(dist - 1, len); - } + len = ReadBits(1) + 3; + dist = DecodeNum(PosHf2); + dist = (dist << 5) | ReadBits(5); + if (dist == 0) + return S_FALSE; + return CopyBlock(dist - 1, len); } + bytePlace--; // bytePlace is [0, 255] } else if (NumHuf++ >= 16 && FlagsCnt == 0) - StMode = 1; + StMode = true; bytePlace &= 0xff; AvrPlc += bytePlace; AvrPlc -= AvrPlc >> 8; - Nhfb+=16; + Nhfb += 16; if (Nhfb > 0xff) { - Nhfb=0x90; + Nhfb = 0x90; Nlzb >>= 1; } - m_UnpackSize --; + m_UnpackSize--; m_OutWindowStream.PutByte((Byte)(ChSet[bytePlace] >> 8)); for (;;) { curByte = ChSet[bytePlace]; newBytePlace = NToPl[curByte++ & 0xff]++; - if ((curByte & 0xff) > 0xa1) - CorrHuff(ChSet, NToPl); - else + if ((curByte & 0xff) <= 0xa1) break; + CorrHuff(ChSet, NToPl); } ChSet[bytePlace] = ChSet[newBytePlace]; @@ -327,7 +340,10 @@ HRESULT CDecoder::HuffDecode() void CDecoder::GetFlagsBuf() { UInt32 flags, newFlagsPlace; - UInt32 flagsPlace = DecodeNum(PosHf2); + UInt32 flagsPlace = DecodeNum(PosHf2); // [0, 256] + + if (flagsPlace >= ARRAY_SIZE(ChSetC)) + return; for (;;) { @@ -343,20 +359,6 @@ void CDecoder::GetFlagsBuf() ChSetC[newFlagsPlace] = flags; } -void CDecoder::InitData() -{ - if (!m_IsSolid) - { - AvrPlcB = AvrLn1 = AvrLn2 = AvrLn3 = NumHuf = Buf60 = 0; - AvrPlc = 0x3500; - MaxDist3 = 0x2001; - Nhfb = Nlzb = 0x80; - } - FlagsCnt = 0; - FlagBuf = 0; - StMode = 0; - LCount = 0; -} void CDecoder::CorrHuff(UInt32 *CharSet,UInt32 *NumToPlace) { @@ -369,110 +371,131 @@ void CDecoder::CorrHuff(UInt32 *CharSet,UInt32 *NumToPlace) NumToPlace[i] = (7 - i) * 32; } -void CDecoder::InitHuff() -{ - for (UInt32 i = 0; i < 256; i++) - { - Place[i] = PlaceA[i] = PlaceB[i] = i; - PlaceC[i] = (~i + 1) & 0xff; - ChSet[i] = ChSetB[i] = i << 8; - ChSetA[i] = i; - ChSetC[i] = ((~i + 1) & 0xff) << 8; - } - memset(NToPl, 0, sizeof(NToPl)); - memset(NToPlB, 0, sizeof(NToPlB)); - memset(NToPlC, 0, sizeof(NToPlC)); - CorrHuff(ChSetB, NToPlB); -} + HRESULT CDecoder::CodeReal(ISequentialInStream *inStream, ISequentialOutStream *outStream, const UInt64 *inSize, const UInt64 *outSize, ICompressProgressInfo * /* progress */) { - if (inSize == NULL || outSize == NULL) + if (!inSize || !outSize) return E_INVALIDARG; + if (_isSolid && !_solidAllowed) + return S_FALSE; + + _solidAllowed = false; + if (!m_OutWindowStream.Create(kHistorySize)) return E_OUTOFMEMORY; if (!m_InBitStream.Create(1 << 20)) return E_OUTOFMEMORY; - m_UnpackSize = (Int64)*outSize; + m_UnpackSize = *outSize; + m_OutWindowStream.SetStream(outStream); - m_OutWindowStream.Init(m_IsSolid); + m_OutWindowStream.Init(_isSolid); m_InBitStream.SetStream(inStream); m_InBitStream.Init(); - // CCoderReleaser coderReleaser(this); - InitData(); - if (!m_IsSolid) + // InitData + + FlagsCnt = 0; + FlagBuf = 0; + StMode = false; + LCount = 0; + + if (!_isSolid) { - InitStructures(); - InitHuff(); + AvrPlcB = AvrLn1 = AvrLn2 = AvrLn3 = NumHuf = Buf60 = 0; + AvrPlc = 0x3500; + MaxDist3 = 0x2001; + Nhfb = Nlzb = 0x80; + + { + // InitStructures + for (int i = 0; i < kNumRepDists; i++) + m_RepDists[i] = 0; + m_RepDistPtr = 0; + LastLength = 0; + LastDist = 0; + } + + // InitHuff + + for (UInt32 i = 0; i < 256; i++) + { + Place[i] = PlaceA[i] = PlaceB[i] = i; + UInt32 c = (~i + 1) & 0xff; + PlaceC[i] = c; + ChSet[i] = ChSetB[i] = i << 8; + ChSetA[i] = i; + ChSetC[i] = c << 8; + } + memset(NToPl, 0, sizeof(NToPl)); + memset(NToPlB, 0, sizeof(NToPlB)); + memset(NToPlC, 0, sizeof(NToPlC)); + CorrHuff(ChSetB, NToPlB); } + if (m_UnpackSize > 0) { GetFlagsBuf(); FlagsCnt = 8; } - while (m_UnpackSize > 0) + while (m_UnpackSize != 0) { - if (StMode) + if (!StMode) { - RINOK(HuffDecode()); - continue; - } - - if (--FlagsCnt < 0) - { - GetFlagsBuf(); - FlagsCnt=7; - } - - if (FlagBuf & 0x80) - { - FlagBuf <<= 1; - if (Nlzb > Nhfb) - { - RINOK(LongLZ()); - } - else - { - RINOK(HuffDecode()); - } - } - else - { - FlagBuf <<= 1; if (--FlagsCnt < 0) { GetFlagsBuf(); FlagsCnt = 7; } + if (FlagBuf & 0x80) { FlagBuf <<= 1; if (Nlzb > Nhfb) - { - RINOK(HuffDecode()); - } - else { RINOK(LongLZ()); + continue; } } else { FlagBuf <<= 1; - RINOK(ShortLZ()); + + if (--FlagsCnt < 0) + { + GetFlagsBuf(); + FlagsCnt = 7; + } + + if ((FlagBuf & 0x80) == 0) + { + FlagBuf <<= 1; + RINOK(ShortLZ()); + continue; + } + + FlagBuf <<= 1; + + if (Nlzb <= Nhfb) + { + RINOK(LongLZ()); + continue; + } } } + + RINOK(HuffDecode()); } - if (m_UnpackSize < 0) - return S_FALSE; + + _solidAllowed = true; return m_OutWindowStream.Flush(); } + STDMETHODIMP CDecoder::Code(ISequentialInStream *inStream, ISequentialOutStream *outStream, const UInt64 *inSize, const UInt64 *outSize, ICompressProgressInfo *progress) { @@ -486,7 +509,7 @@ STDMETHODIMP CDecoder::SetDecoderProperties2(const Byte *data, UInt32 size) { if (size < 1) return E_INVALIDARG; - m_IsSolid = ((data[0] & 1) != 0); + _isSolid = ((data[0] & 1) != 0); return S_OK; } diff --git a/CPP/7zip/Compress/Rar1Decoder.h b/CPP/7zip/Compress/Rar1Decoder.h index 5f1bd2de..b5dc1eb4 100644 --- a/CPP/7zip/Compress/Rar1Decoder.h +++ b/CPP/7zip/Compress/Rar1Decoder.h @@ -20,48 +20,45 @@ namespace NRar1 { const UInt32 kNumRepDists = 4; -typedef NBitm::CDecoder CBitDecoder; - class CDecoder : public ICompressCoder, public ICompressSetDecoderProperties2, public CMyUnknownImp { -public: CLzOutWindow m_OutWindowStream; - CBitDecoder m_InBitStream; + NBitm::CDecoder m_InBitStream; - UInt32 m_RepDists[kNumRepDists]; - UInt32 m_RepDistPtr; + UInt64 m_UnpackSize; UInt32 LastDist; UInt32 LastLength; - Int64 m_UnpackSize; - bool m_IsSolid; + UInt32 m_RepDistPtr; + UInt32 m_RepDists[kNumRepDists]; - UInt32 ReadBits(int numBits); + bool _isSolid; + bool _solidAllowed; + + bool StMode; + int FlagsCnt; + UInt32 FlagBuf, AvrPlc, AvrPlcB, AvrLn1, AvrLn2, AvrLn3; + unsigned Buf60, NumHuf, LCount; + UInt32 Nhfb, Nlzb, MaxDist3; + + UInt32 ChSet[256], ChSetA[256], ChSetB[256], ChSetC[256]; + UInt32 Place[256], PlaceA[256], PlaceB[256], PlaceC[256]; + UInt32 NToPl[256], NToPlB[256], NToPlC[256]; + + UInt32 ReadBits(unsigned numBits); HRESULT CopyBlock(UInt32 distance, UInt32 len); - - UInt32 DecodeNum(const UInt32 *posTab); + UInt32 DecodeNum(const Byte *numTab); HRESULT ShortLZ(); HRESULT LongLZ(); HRESULT HuffDecode(); void GetFlagsBuf(); - void InitData(); - void InitHuff(); void CorrHuff(UInt32 *CharSet, UInt32 *NumToPlace); void OldUnpWriteBuf(); - UInt32 ChSet[256],ChSetA[256],ChSetB[256],ChSetC[256]; - UInt32 Place[256],PlaceA[256],PlaceB[256],PlaceC[256]; - UInt32 NToPl[256],NToPlB[256],NToPlC[256]; - UInt32 FlagBuf,AvrPlc,AvrPlcB,AvrLn1,AvrLn2,AvrLn3; - int Buf60,NumHuf,StMode,LCount,FlagsCnt; - UInt32 Nhfb,Nlzb,MaxDist3; - - void InitStructures(); - HRESULT CodeReal(ISequentialInStream *inStream, ISequentialOutStream *outStream, const UInt64 *inSize, const UInt64 *outSize, ICompressProgressInfo *progress); @@ -70,14 +67,6 @@ public: MY_UNKNOWN_IMP1(ICompressSetDecoderProperties2) - /* - void ReleaseStreams() - { - m_OutWindowStream.ReleaseStream(); - m_InBitStream.ReleaseStream(); - } - */ - STDMETHOD(Code)(ISequentialInStream *inStream, ISequentialOutStream *outStream, const UInt64 *inSize, const UInt64 *outSize, ICompressProgressInfo *progress); diff --git a/CPP/7zip/Compress/Rar2Decoder.cpp b/CPP/7zip/Compress/Rar2Decoder.cpp index 727ff8d9..dd27e369 100644 --- a/CPP/7zip/Compress/Rar2Decoder.cpp +++ b/CPP/7zip/Compress/Rar2Decoder.cpp @@ -80,7 +80,8 @@ static const UInt32 kHistorySize = 1 << 20; static const UInt32 kWindowReservSize = (1 << 22) + 256; CDecoder::CDecoder(): - m_IsSolid(false), + _isSolid(false), + _solidAllowed(false), m_TablesOK(false) { } @@ -130,7 +131,7 @@ bool CDecoder::ReadTables(void) i = 0; - while (i < numLevels) + do { UInt32 sym = m_LevelDecoder.Decode(&m_InBitStream); if (sym < kTableDirectLevels) @@ -144,10 +145,7 @@ bool CDecoder::ReadTables(void) { unsigned num = ReadBits(2) + 3; if (i == 0) - { - // return false; - continue; // original unRAR - } + return false; num += i; if (num > numLevels) { @@ -180,6 +178,10 @@ bool CDecoder::ReadTables(void) } } } + while (i < numLevels); + + if (m_InBitStream.ExtraBitsWereRead()) + return false; if (m_AudioMode) for (i = 0; i < m_NumChannels; i++) @@ -226,24 +228,14 @@ bool CDecoder::ReadLastTables() return true; } -/* -class CCoderReleaser -{ - CDecoder *m_Coder; -public: - CCoderReleaser(CDecoder *coder): m_Coder(coder) {} - ~CCoderReleaser() - { - m_Coder->ReleaseStreams(); - } -}; -*/ bool CDecoder::DecodeMm(UInt32 pos) { while (pos-- != 0) { UInt32 symbol = m_MMDecoders[m_MmFilter.CurrentChannel].Decode(&m_InBitStream); + if (m_InBitStream.ExtraBitsWereRead()) + return false; if (symbol >= 256) return symbol == 256; /* @@ -264,6 +256,8 @@ bool CDecoder::DecodeLz(Int32 pos) while (pos > 0) { UInt32 sym = m_MainDecoder.Decode(&m_InBitStream); + if (m_InBitStream.ExtraBitsWereRead()) + return false; UInt32 length, distance; if (sym < 256) { @@ -338,9 +332,13 @@ bool CDecoder::DecodeLz(Int32 pos) HRESULT CDecoder::CodeReal(ISequentialInStream *inStream, ISequentialOutStream *outStream, const UInt64 *inSize, const UInt64 *outSize, ICompressProgressInfo *progress) { - if (inSize == NULL || outSize == NULL) + if (!inSize || !outSize) return E_INVALIDARG; + if (_isSolid && !_solidAllowed) + return S_FALSE; + _solidAllowed = false; + if (!m_OutWindowStream.Create(kHistorySize)) return E_OUTOFMEMORY; if (!m_InBitStream.Create(1 << 20)) @@ -351,12 +349,12 @@ HRESULT CDecoder::CodeReal(ISequentialInStream *inStream, ISequentialOutStream * UInt64 pos = 0, unPackSize = *outSize; m_OutWindowStream.SetStream(outStream); - m_OutWindowStream.Init(m_IsSolid); + m_OutWindowStream.Init(_isSolid); m_InBitStream.SetStream(inStream); m_InBitStream.Init(); // CCoderReleaser coderReleaser(this); - if (!m_IsSolid) + if (!_isSolid) { InitStructures(); if (unPackSize == 0) @@ -364,6 +362,7 @@ HRESULT CDecoder::CodeReal(ISequentialInStream *inStream, ISequentialOutStream * if (m_InBitStream.GetProcessedSize() + 2 <= m_PackSize) // test it: probably incorrect; if (!ReadTables()) return S_FALSE; + _solidAllowed = true; return S_OK; } ReadTables(); @@ -389,15 +388,19 @@ HRESULT CDecoder::CodeReal(ISequentialInStream *inStream, ISequentialOutStream * if (!DecodeLz((Int32)blockSize)) return S_FALSE; } + + if (m_InBitStream.ExtraBitsWereRead()) + return S_FALSE; + UInt64 globalPos = m_OutWindowStream.GetProcessedSize(); pos = globalPos - blockStartPos; if (pos < blockSize) if (!ReadTables()) return S_FALSE; pos = globalPos - startPos; - if (progress != 0) + if (progress) { - UInt64 packSize = m_InBitStream.GetProcessedSize(); + const UInt64 packSize = m_InBitStream.GetProcessedSize(); RINOK(progress->SetRatioInfo(&packSize, &pos)); } } @@ -406,6 +409,9 @@ HRESULT CDecoder::CodeReal(ISequentialInStream *inStream, ISequentialOutStream * if (!ReadLastTables()) return S_FALSE; + + _solidAllowed = true; + return m_OutWindowStream.Flush(); } @@ -422,7 +428,7 @@ STDMETHODIMP CDecoder::SetDecoderProperties2(const Byte *data, UInt32 size) { if (size < 1) return E_INVALIDARG; - m_IsSolid = ((data[0] & 1) != 0); + _isSolid = ((data[0] & 1) != 0); return S_OK; } diff --git a/CPP/7zip/Compress/Rar2Decoder.h b/CPP/7zip/Compress/Rar2Decoder.h index 939538e8..a8a531d8 100644 --- a/CPP/7zip/Compress/Rar2Decoder.h +++ b/CPP/7zip/Compress/Rar2Decoder.h @@ -125,7 +125,8 @@ class CDecoder : UInt32 m_LastLength; - bool m_IsSolid; + bool _isSolid; + bool _solidAllowed; bool m_TablesOK; bool m_AudioMode; @@ -159,14 +160,6 @@ public: MY_UNKNOWN_IMP1(ICompressSetDecoderProperties2) - /* - void ReleaseStreams() - { - m_OutWindowStream.ReleaseStream(); - m_InBitStream.ReleaseStream(); - } - */ - STDMETHOD(Code)(ISequentialInStream *inStream, ISequentialOutStream *outStream, const UInt64 *inSize, const UInt64 *outSize, ICompressProgressInfo *progress); diff --git a/CPP/7zip/Compress/Rar3Decoder.cpp b/CPP/7zip/Compress/Rar3Decoder.cpp index 78ac0bf9..a37f9eef 100644 --- a/CPP/7zip/Compress/Rar3Decoder.cpp +++ b/CPP/7zip/Compress/Rar3Decoder.cpp @@ -94,7 +94,8 @@ CDecoder::CDecoder(): _writtenFileSize(0), _vmData(0), _vmCode(0), - m_IsSolid(false) + _isSolid(false), + _solidAllowed(false) { Ppmd7_Construct(&_ppmd); } @@ -135,7 +136,7 @@ HRESULT CDecoder::WriteArea(UInt32 startPtr, UInt32 endPtr) return WriteData(_window, endPtr); } -void CDecoder::ExecuteFilter(int tempFilterIndex, NVm::CBlockRef &outBlockRef) +void CDecoder::ExecuteFilter(unsigned tempFilterIndex, NVm::CBlockRef &outBlockRef) { CTempFilter *tempFilter = _tempFilters[tempFilterIndex]; tempFilter->InitR[6] = (UInt32)_writtenFileSize; @@ -405,7 +406,7 @@ bool CDecoder::ReadVmCodePPM() #define RIF(x) { if (!(x)) return S_FALSE; } -UInt32 CDecoder::ReadBits(int numBits) { return m_InBitStream.BitDecoder.ReadBits(numBits); } +UInt32 CDecoder::ReadBits(unsigned numBits) { return m_InBitStream.BitDecoder.ReadBits(numBits); } // ---------- PPM ---------- @@ -414,7 +415,7 @@ HRESULT CDecoder::InitPPM() unsigned maxOrder = (unsigned)ReadBits(7); bool reset = ((maxOrder & 0x20) != 0); - int maxMB = 0; + UInt32 maxMB = 0; if (reset) maxMB = (Byte)ReadBits(8); else @@ -556,12 +557,13 @@ HRESULT CDecoder::ReadTables(bool &keepDecompressing) PrevAlignCount = 0; Byte levelLevels[kLevelTableSize]; - Byte newLevels[kTablesSizesSum]; + Byte lens[kTablesSizesSum]; if (ReadBits(1) == 0) memset(m_LastLevels, 0, kTablesSizesSum); - int i; + unsigned i; + for (i = 0; i < kLevelTableSize; i++) { UInt32 length = ReadBits(4); @@ -579,39 +581,44 @@ HRESULT CDecoder::ReadTables(bool &keepDecompressing) } levelLevels[i] = (Byte)length; } + RIF(m_LevelDecoder.Build(levelLevels)); + i = 0; - while (i < kTablesSizesSum) + + do { UInt32 sym = m_LevelDecoder.Decode(&m_InBitStream.BitDecoder); if (sym < 16) { - newLevels[i] = Byte((sym + m_LastLevels[i]) & 15); + lens[i] = Byte((sym + m_LastLevels[i]) & 15); i++; } else if (sym > kLevelTableSize) return S_FALSE; else { - int num; - if (((sym - 16) & 1) == 0) - num = ReadBits(3) + 3; - else - num = ReadBits(7) + 11; - if (sym < 18) + unsigned num = ((sym - 16) & 1) * 4; + num += num + 3 + (unsigned)ReadBits(num + 3); + num += i; + if (num > kTablesSizesSum) + num = kTablesSizesSum; + Byte v = 0; + if (sym < 16 + 2) { if (i == 0) return S_FALSE; - for (; num > 0 && i < kTablesSizesSum; num--, i++) - newLevels[i] = newLevels[(size_t)i - 1]; - } - else - { - for (; num > 0 && i < kTablesSizesSum; num--) - newLevels[i++] = 0; + v = lens[(size_t)i - 1]; } + do + lens[i++] = v; + while (i < num); } } + while (i < kTablesSizesSum); + + if (InputEofError()) + return S_FALSE; TablesRead = true; @@ -624,12 +631,12 @@ HRESULT CDecoder::ReadTables(bool &keepDecompressing) } */ - RIF(m_MainDecoder.Build(&newLevels[0])); - RIF(m_DistDecoder.Build(&newLevels[kMainTableSize])); - RIF(m_AlignDecoder.Build(&newLevels[kMainTableSize + kDistTableSize])); - RIF(m_LenDecoder.Build(&newLevels[kMainTableSize + kDistTableSize + kAlignTableSize])); + RIF(m_MainDecoder.Build(&lens[0])); + RIF(m_DistDecoder.Build(&lens[kMainTableSize])); + RIF(m_AlignDecoder.Build(&lens[kMainTableSize + kDistTableSize])); + RIF(m_LenDecoder.Build(&lens[kMainTableSize + kDistTableSize + kAlignTableSize])); - memcpy(m_LastLevels, newLevels, kTablesSizesSum); + memcpy(m_LastLevels, lens, kTablesSizesSum); TablesOK = true; @@ -769,7 +776,7 @@ HRESULT CDecoder::DecodeLZ(bool &keepDecompressing) if (sym2 >= kDistTableSize) return S_FALSE; rep0 = kDistStart[sym2]; - int numBits = kDistDirectBits[sym2]; + unsigned numBits = kDistDirectBits[sym2]; if (sym2 >= (kNumAlignBits * 2) + 2) { if (numBits > kNumAlignBits) @@ -822,7 +829,7 @@ HRESULT CDecoder::CodeReal(ICompressProgressInfo *progress) _writtenFileSize = 0; _unsupportedFilter = false; - if (!m_IsSolid) + if (!_isSolid) { _lzSize = 0; _winPos = 0; @@ -835,14 +842,23 @@ HRESULT CDecoder::CodeReal(ICompressProgressInfo *progress) PpmEscChar = 2; PpmError = true; InitFilters(); + // _errorMode = false; } - if (!m_IsSolid || !TablesRead) + /* + if (_errorMode) + return S_FALSE; + */ + + if (!_isSolid || !TablesRead) { bool keepDecompressing; RINOK(ReadTables(keepDecompressing)); if (!keepDecompressing) + { + _solidAllowed = true; return S_OK; + } } for (;;) @@ -867,6 +883,9 @@ HRESULT CDecoder::CodeReal(ICompressProgressInfo *progress) if (!keepDecompressing) break; } + + _solidAllowed = true; + RINOK(WriteBuf()); UInt64 packSize = m_InBitStream.BitDecoder.GetProcessedSize(); RINOK(progress->SetRatioInfo(&packSize, &_writtenFileSize)); @@ -887,6 +906,10 @@ STDMETHODIMP CDecoder::Code(ISequentialInStream *inStream, ISequentialOutStream if (!inSize) return E_INVALIDARG; + if (_isSolid && !_solidAllowed) + return S_FALSE; + _solidAllowed = false; + if (!_vmData) { _vmData = (Byte *)::MidAlloc(kVmDataSizeMax + kVmCodeSizeMax); @@ -915,8 +938,8 @@ STDMETHODIMP CDecoder::Code(ISequentialInStream *inStream, ISequentialOutStream _unpackSize = outSize ? *outSize : (UInt64)(Int64)-1; return CodeReal(progress); } - catch(const CInBufferException &e) { return e.ErrorCode; } - catch(...) { return S_FALSE; } + catch(const CInBufferException &e) { /* _errorMode = true; */ return e.ErrorCode; } + catch(...) { /* _errorMode = true; */ return S_FALSE; } // CNewException is possible here. But probably CNewException is caused // by error in data stream. } @@ -925,7 +948,7 @@ STDMETHODIMP CDecoder::SetDecoderProperties2(const Byte *data, UInt32 size) { if (size < 1) return E_INVALIDARG; - m_IsSolid = ((data[0] & 1) != 0); + _isSolid = ((data[0] & 1) != 0); return S_OK; } diff --git a/CPP/7zip/Compress/Rar3Decoder.h b/CPP/7zip/Compress/Rar3Decoder.h index 4cd2bd14..50462987 100644 --- a/CPP/7zip/Compress/Rar3Decoder.h +++ b/CPP/7zip/Compress/Rar3Decoder.h @@ -31,7 +31,7 @@ const UInt32 kLenTableSize = 28; const UInt32 kMainTableSize = 256 + 1 + 1 + 1 + kNumReps + kNumLen2Symbols + kLenTableSize; const UInt32 kDistTableSize = 60; -const int kNumAlignBits = 4; +const unsigned kNumAlignBits = 4; const UInt32 kAlignTableSize = (1 << kNumAlignBits) + 1; const UInt32 kLevelTableSize = 20; @@ -158,7 +158,7 @@ struct CTempFilter: public NVm::CProgramInitState } }; -const int kNumHuffmanBits = 15; +const unsigned kNumHuffmanBits = 15; class CDecoder: public ICompressCoder, @@ -191,7 +191,9 @@ class CDecoder: CRecordVector _tempFilters; UInt32 _lastFilter; - bool m_IsSolid; + bool _isSolid; + bool _solidAllowed; + // bool _errorMode; bool _lzMode; bool _unsupportedFilter; @@ -209,7 +211,7 @@ class CDecoder: HRESULT WriteDataToStream(const Byte *data, UInt32 size); HRESULT WriteData(const Byte *data, UInt32 size); HRESULT WriteArea(UInt32 startPtr, UInt32 endPtr); - void ExecuteFilter(int tempFilterIndex, NVm::CBlockRef &outBlockRef); + void ExecuteFilter(unsigned tempFilterIndex, NVm::CBlockRef &outBlockRef); HRESULT WriteBuf(); void InitFilters(); @@ -217,7 +219,7 @@ class CDecoder: bool ReadVmCodeLZ(); bool ReadVmCodePPM(); - UInt32 ReadBits(int numBits); + UInt32 ReadBits(unsigned numBits); HRESULT InitPPM(); int DecodePpmSymbol(); diff --git a/CPP/7zip/Compress/Rar5Decoder.cpp b/CPP/7zip/Compress/Rar5Decoder.cpp index dbc94d63..7c963b10 100644 --- a/CPP/7zip/Compress/Rar5Decoder.cpp +++ b/CPP/7zip/Compress/Rar5Decoder.cpp @@ -72,6 +72,7 @@ CDecoder::CDecoder(): _writtenFileSize(0), _dictSizeLog(0), _isSolid(false), + _solidAllowed(false), _wasInit(false), _inputBuf(NULL) { @@ -334,58 +335,63 @@ HRESULT CDecoder::ReadTables(CBitDecoder &_bitStream) { if (_progress) { - UInt64 packSize = _bitStream.GetProcessedSize(); + const UInt64 packSize = _bitStream.GetProcessedSize(); RINOK(_progress->SetRatioInfo(&packSize, &_writtenFileSize)); } _bitStream.AlignToByte(); _bitStream.Prepare(); - unsigned flags = _bitStream.ReadByteInAligned(); - unsigned checkSum = _bitStream.ReadByteInAligned(); - checkSum ^= flags; - - UInt32 blockSize; { + unsigned flags = _bitStream.ReadByteInAligned(); + unsigned checkSum = _bitStream.ReadByteInAligned(); + checkSum ^= flags; unsigned num = (flags >> 3) & 3; if (num == 3) return S_FALSE; - blockSize = _bitStream.ReadByteInAligned(); - if (num > 0) + UInt32 blockSize = _bitStream.ReadByteInAligned(); + checkSum ^= blockSize; + + if (num != 0) { - blockSize += (UInt32)_bitStream.ReadByteInAligned() << 8; + unsigned b = _bitStream.ReadByteInAligned(); + checkSum ^= b; + blockSize += (UInt32)b << 8; if (num > 1) - blockSize += (UInt32)_bitStream.ReadByteInAligned() << 16; + { + b = _bitStream.ReadByteInAligned(); + checkSum ^= b; + blockSize += (UInt32)b << 16; + } } - } - - checkSum ^= blockSize ^ (blockSize >> 8) ^ (blockSize >> 16); - if ((Byte)checkSum != 0x5A) - return S_FALSE; - - unsigned blockSizeBits7 = (flags & 7) + 1; - - if (blockSize == 0 && blockSizeBits7 != 8) - return S_FALSE; - - blockSize += (blockSizeBits7 >> 3); - blockSize--; - - _bitStream._blockEndBits7 = (Byte)(blockSizeBits7 & 7); - _bitStream._blockEnd = _bitStream.GetProcessedSize_Round() + blockSize; - - _bitStream.SetCheck2(); - - _isLastBlock = ((flags & 0x40) != 0); - - if ((flags & 0x80) == 0) - { - if (!_tableWasFilled && blockSize != 0) + + if (checkSum != 0x5A) return S_FALSE; - return S_OK; - } - _tableWasFilled = false; + unsigned blockSizeBits7 = (flags & 7) + 1; + blockSize += (blockSizeBits7 >> 3); + if (blockSize == 0) + return S_FALSE; + blockSize--; + blockSizeBits7 &= 7; + + _bitStream._blockEndBits7 = (Byte)blockSizeBits7; + _bitStream._blockEnd = _bitStream.GetProcessedSize_Round() + blockSize; + + _bitStream.SetCheck2(); + + _isLastBlock = ((flags & 0x40) != 0); + + if ((flags & 0x80) == 0) + { + if (!_tableWasFilled) + if (blockSize != 0 || blockSizeBits7 != 0) + return S_FALSE; + return S_OK; + } + + _tableWasFilled = false; + } { Byte lens2[kLevelTableSize]; @@ -421,7 +427,7 @@ HRESULT CDecoder::ReadTables(CBitDecoder &_bitStream) Byte lens[kTablesSizesSum]; unsigned i = 0; - while (i < kTablesSizesSum) + do { if (_bitStream._buf >= _bitStream._bufCheck2) { @@ -439,34 +445,24 @@ HRESULT CDecoder::ReadTables(CBitDecoder &_bitStream) return S_FALSE; else { - sym -= 16; - unsigned sh = ((sym & 1) << 2); - unsigned num = (unsigned)_bitStream.ReadBits9(3 + sh) + 3 + (sh << 1); - + unsigned num = ((sym - 16) & 1) * 4; + num += num + 3 + (unsigned)_bitStream.ReadBits9(num + 3); num += i; if (num > kTablesSizesSum) num = kTablesSizesSum; - - if (sym < 2) + Byte v = 0; + if (sym < 16 + 2) { if (i == 0) - { - // return S_FALSE; - continue; // original unRAR - } - Byte v = lens[(size_t)i - 1]; - do - lens[i++] = v; - while (i < num); - } - else - { - do - lens[i++] = 0; - while (i < num); + return S_FALSE; + v = lens[(size_t)i - 1]; } + do + lens[i++] = v; + while (i < num); } } + while (i < kTablesSizesSum); if (_bitStream.IsBlockOverRead()) return S_FALSE; @@ -606,6 +602,10 @@ HRESULT CDecoder::DecodeLZ() } } } + + // that check is not required, but it can help, if there is BUG in another code + if (!_tableWasFilled) + break; // return S_FALSE; } UInt32 sym = m_MainDecoder.Decode(&_bitStream); @@ -807,7 +807,10 @@ HRESULT CDecoder::CodeReal() */ if (res == S_OK) + { + _solidAllowed = true; res = res2; + } if (res == S_OK && _unpackSize_Defined && _writtenFileSize != _unpackSize) return S_FALSE; @@ -827,6 +830,10 @@ STDMETHODIMP CDecoder::Code(ISequentialInStream *inStream, ISequentialOutStream { try { + if (_isSolid && !_solidAllowed) + return S_FALSE; + _solidAllowed = false; + if (_dictSizeLog >= sizeof(size_t) * 8) return E_NOTIMPL; diff --git a/CPP/7zip/Compress/Rar5Decoder.h b/CPP/7zip/Compress/Rar5Decoder.h index fc863ea0..9039b481 100644 --- a/CPP/7zip/Compress/Rar5Decoder.h +++ b/CPP/7zip/Compress/Rar5Decoder.h @@ -127,7 +127,7 @@ public: return *_buf++; } - UInt32 GetValue(unsigned numBits) + UInt32 GetValue(unsigned numBits) const { UInt32 v = ((UInt32)_buf[0] << 16) | ((UInt32)_buf[1] << 8) | (UInt32)_buf[2]; v >>= (24 - numBits - _bitPos); @@ -218,6 +218,13 @@ class CDecoder: bool _unsupportedFilter; bool _lzError; bool _writeError; + + bool _isSolid; + bool _solidAllowed; + bool _tableWasFilled; + bool _wasInit; + + Byte _dictSizeLog; // CBitDecoder _bitStream; Byte *_window; @@ -238,11 +245,6 @@ class CDecoder: UInt64 _writtenFileSize; size_t _winSizeAllocated; - Byte _dictSizeLog; - bool _tableWasFilled; - bool _isSolid; - bool _wasInit; - UInt32 _reps[kNumReps]; UInt32 _lastLen; diff --git a/CPP/7zip/Compress/ShrinkDecoder.cpp b/CPP/7zip/Compress/ShrinkDecoder.cpp index 03ac7dc9..3f3a42a3 100644 --- a/CPP/7zip/Compress/ShrinkDecoder.cpp +++ b/CPP/7zip/Compress/ShrinkDecoder.cpp @@ -13,6 +13,7 @@ namespace NCompress { namespace NShrink { +static const UInt32 kEmpty = 256; // kNumItems; static const UInt32 kBufferSize = (1 << 18); static const unsigned kNumMinBits = 9; @@ -34,20 +35,15 @@ HRESULT CDecoder::CodeReal(ISequentialInStream *inStream, ISequentialOutStream * outBuffer.Init(); { - unsigned i; - for (i = 0; i < 257; i++) - _parents[i] = (UInt16)i; - for (; i < kNumItems; i++) - _parents[i] = kNumItems; - for (i = 0; i < kNumItems; i++) - _suffixes[i] = 0; + for (unsigned i = 0; i < kNumItems; i++) + _parents[i] = kEmpty; } - UInt64 prevPos = 0, inPrev = 0; + UInt64 outPrev = 0, inPrev = 0; unsigned numBits = kNumMinBits; unsigned head = 257; int lastSym = -1; - Byte lastChar2 = 0; + Byte lastChar = 0; bool moreOut = false; HRESULT res = S_FALSE; @@ -67,18 +63,22 @@ HRESULT CDecoder::CodeReal(ISequentialInStream *inStream, ISequentialOutStream * break; } eofCheck = true; - // Is specSym(=256) allowed after end of stream - // Do we need to read it here + // Is specSym(=256) allowed after end of stream ? + // Do we need to read it here ? } if (progress) { - if (nowPos - prevPos >= (1 << 18) - || _inProcessed - inPrev >= (1 << 20)) + if (nowPos - outPrev >= (1 << 20) || _inProcessed - inPrev >= (1 << 20)) { - prevPos = nowPos; + outPrev = nowPos; inPrev = _inProcessed; - RINOK(progress->SetRatioInfo(&_inProcessed, &nowPos)); + res = progress->SetRatioInfo(&_inProcessed, &nowPos); + if (res != SZ_OK) + { + // break; + return res; + } } } @@ -105,23 +105,30 @@ HRESULT CDecoder::CodeReal(ISequentialInStream *inStream, ISequentialOutStream * continue; } if (sym != 2) - break; { + break; + // continue; // info-zip just ignores such code + } + { + /* + ---------- Free leaf nodes ---------- + Note : that code can mark _parents[lastSym] as free, and next + inserted node will be Orphan in that case. + */ + unsigned i; - for (i = 257; i < kNumItems; i++) + for (i = 256; i < kNumItems; i++) _stack[i] = 0; for (i = 257; i < kNumItems; i++) { unsigned par = _parents[i]; - if (par != kNumItems) + if (par != kEmpty) _stack[par] = 1; } for (i = 257; i < kNumItems; i++) if (_stack[i] == 0) - _parents[i] = kNumItems; - + _parents[i] = kEmpty; head = 257; - continue; } } @@ -137,27 +144,22 @@ HRESULT CDecoder::CodeReal(ISequentialInStream *inStream, ISequentialOutStream * bool needPrev = false; if (head < kNumItems && lastSym >= 0) { - while (head < kNumItems && _parents[head] != kNumItems) + while (head < kNumItems && _parents[head] != kEmpty) head++; if (head < kNumItems) { - if (head == (unsigned)lastSym) - { - // we need to fix the code for that case - // _parents[head] is not allowed to link to itself - res = E_NOTIMPL; - break; - } + /* + if (head == lastSym), it updates Orphan to self-linked Orphan and creates two problems: + 1) we must check _stack[i++] overflow in code that walks tree nodes. + 2) self-linked node can not be removed. So such self-linked nodes can occupy all _parents items. + */ needPrev = true; _parents[head] = (UInt16)lastSym; - _suffixes[head] = (Byte)lastChar2; + _suffixes[head] = (Byte)lastChar; head++; } } - if (_parents[sym] == kNumItems) - break; - lastSym = sym; unsigned cur = sym; unsigned i = 0; @@ -166,10 +168,17 @@ HRESULT CDecoder::CodeReal(ISequentialInStream *inStream, ISequentialOutStream * { _stack[i++] = _suffixes[cur]; cur = _parents[cur]; + // don't change that code: + // Orphan Check and self-linked Orphan check (_stack overflow check); + if (cur == kEmpty || i >= kNumItems) + break; } + if (cur == kEmpty || i >= kNumItems) + break; + _stack[i++] = (Byte)cur; - lastChar2 = (Byte)cur; + lastChar = (Byte)cur; if (needPrev) _suffixes[(size_t)head - 1] = (Byte)cur; diff --git a/CPP/7zip/Compress/ShrinkDecoder.h b/CPP/7zip/Compress/ShrinkDecoder.h index c3a3c624..ed3f2ddc 100644 --- a/CPP/7zip/Compress/ShrinkDecoder.h +++ b/CPP/7zip/Compress/ShrinkDecoder.h @@ -19,8 +19,8 @@ class CDecoder : public ICompressGetInStreamProcessedSize, public CMyUnknownImp { - UInt64 _inProcessed; bool _fullStreamMode; + UInt64 _inProcessed; UInt16 _parents[kNumItems]; Byte _suffixes[kNumItems]; diff --git a/CPP/7zip/Compress/XzDecoder.cpp b/CPP/7zip/Compress/XzDecoder.cpp index 51ecc4ee..4fcd09fe 100644 --- a/CPP/7zip/Compress/XzDecoder.cpp +++ b/CPP/7zip/Compress/XzDecoder.cpp @@ -4,248 +4,119 @@ #include "../../../C/Alloc.h" -#include "../Common/StreamUtils.h" - -#include "../Archive/IArchive.h" +#include "../Common/CWrappers.h" #include "XzDecoder.h" -using namespace NArchive; - namespace NCompress { namespace NXz { +#define RET_IF_WRAP_ERROR_CONFIRMED(wrapRes, sRes, sResErrorCode) \ + if (wrapRes != S_OK && sRes == sResErrorCode) return wrapRes; -CXzUnpackerCPP::CXzUnpackerCPP(): InBuf(NULL), OutBuf(NULL) +#define RET_IF_WRAP_ERROR(wrapRes, sRes, sResErrorCode) \ + if (wrapRes != S_OK /* && (sRes == SZ_OK || sRes == sResErrorCode) */) return wrapRes; + +static HRESULT SResToHRESULT_Code(SRes res) throw() { - XzUnpacker_Construct(&p, &g_Alloc); -} - -CXzUnpackerCPP::~CXzUnpackerCPP() -{ - XzUnpacker_Free(&p); - MidFree(InBuf); - MidFree(OutBuf); -} - - -void CStatInfo::Clear() -{ - InSize = 0; - OutSize = 0; - PhySize = 0; - - NumStreams = 0; - NumBlocks = 0; - - UnpackSize_Defined = false; - - NumStreams_Defined = false; - NumBlocks_Defined = false; - - IsArc = false; - UnexpectedEnd = false; - DataAfterEnd = false; - Unsupported = false; - HeadersError = false; - DataError = false; - CrcError = false; + if (res < 0) + return res; + switch (res) + { + case SZ_OK: return S_OK; + case SZ_ERROR_MEM: return E_OUTOFMEMORY; + case SZ_ERROR_UNSUPPORTED: return E_NOTIMPL; + } + return S_FALSE; } HRESULT CDecoder::Decode(ISequentialInStream *seqInStream, ISequentialOutStream *outStream, const UInt64 *outSizeLimit, bool finishStream, ICompressProgressInfo *progress) { - const size_t kInBufSize = (size_t)1 << 20; - const size_t kOutBufSize = (size_t)1 << 21; + MainDecodeSRes = S_OK; + MainDecodeSRes_wasUsed = false; + XzStatInfo_Clear(&Stat); - Clear(); - DecodeRes = SZ_OK; - - XzUnpacker_Init(&xzu.p); - - if (!xzu.InBuf) + if (!xz) { - xzu.InBuf = (Byte *)MidAlloc(kInBufSize); - if (!xzu.InBuf) + xz = XzDecMt_Create(&g_Alloc, &g_MidAlloc); + if (!xz) return E_OUTOFMEMORY; } - if (!xzu.OutBuf) - { - xzu.OutBuf = (Byte *)MidAlloc(kOutBufSize); - if (!xzu.OutBuf) - return E_OUTOFMEMORY; - } - - UInt32 inSize = 0; - UInt32 inPos = 0; - SizeT outPos = 0; - HRESULT readRes = S_OK; + CXzDecMtProps props; + XzDecMtProps_Init(&props); - for (;;) + int isMT = False; + + #ifndef _7ZIP_ST { - if (inPos == inSize && readRes == S_OK) + props.numThreads = 1; + UInt32 numThreads = _numThreads; + + if (_tryMt && numThreads > 1) { - inPos = inSize = 0; - readRes = seqInStream->Read(xzu.InBuf, kInBufSize, &inSize); + size_t memUsage = (size_t)_memUsage; + if (memUsage != _memUsage) + memUsage = (size_t)0 - 1; + props.memUseMax = memUsage; + isMT = (numThreads > 1); } - SizeT inLen = inSize - inPos; - SizeT outLen = kOutBufSize - outPos; - ECoderFinishMode finishMode = CODER_FINISH_ANY; + props.numThreads = numThreads; + } + #endif + CSeqInStreamWrap inWrap; + CSeqOutStreamWrap outWrap; + CCompressProgressWrap progressWrap; + + inWrap.Init(seqInStream); + outWrap.Init(outStream); + progressWrap.Init(progress); + + SRes res = XzDecMt_Decode(xz, + &props, + outSizeLimit, finishStream, + &outWrap.vt, + &inWrap.vt, + &Stat, + &isMT, + progress ? &progressWrap.vt : NULL); + + MainDecodeSRes = res; + + #ifndef _7ZIP_ST + // _tryMt = isMT; + #endif + + RET_IF_WRAP_ERROR(outWrap.Res, res, SZ_ERROR_WRITE) + RET_IF_WRAP_ERROR(progressWrap.Res, res, SZ_ERROR_PROGRESS) + RET_IF_WRAP_ERROR_CONFIRMED(inWrap.Res, res, SZ_ERROR_READ) + + // return E_OUTOFMEMORY; + + MainDecodeSRes_wasUsed = true; + + if (res == SZ_OK && finishStream) + { /* - // 17.01 : the code was disabled: - if (inSize == 0) - finishMode = CODER_FINISH_END; + if (inSize && *inSize != Stat.PhySize) + res = SZ_ERROR_DATA; */ - - if (outSizeLimit) - { - const UInt64 rem = *outSizeLimit - OutSize; - if (outLen >= rem) - { - outLen = (SizeT)rem; - if (finishStream) - finishMode = CODER_FINISH_END; - } - } - - ECoderStatus status; - - const SizeT outLenRequested = outLen; - - SRes res = XzUnpacker_Code(&xzu.p, - xzu.OutBuf + outPos, &outLen, - xzu.InBuf + inPos, &inLen, - finishMode, &status); - - DecodeRes = res; - - inPos += (UInt32)inLen; - outPos += outLen; - - InSize += inLen; - OutSize += outLen; - - bool finished = ((inLen == 0 && outLen == 0) || res != SZ_OK); - - if (outLen >= outLenRequested || finished) - { - if (outStream && outPos != 0) - { - RINOK(WriteStream(outStream, xzu.OutBuf, outPos)); - } - outPos = 0; - } - - if (progress) - { - RINOK(progress->SetRatioInfo(&InSize, &OutSize)); - } - - if (!finished) - continue; - - { - PhySize = InSize; - NumStreams = xzu.p.numStartedStreams; - if (NumStreams > 0) - IsArc = true; - NumBlocks = xzu.p.numTotalBlocks; - - UnpackSize_Defined = true; - NumStreams_Defined = true; - NumBlocks_Defined = true; - - UInt64 extraSize = XzUnpacker_GetExtraSize(&xzu.p); - - if (res == SZ_OK) - { - if (status == CODER_STATUS_NEEDS_MORE_INPUT) - { - extraSize = 0; - if (!XzUnpacker_IsStreamWasFinished(&xzu.p)) - { - // finished at padding bytes, but padding is not aligned for 4 - UnexpectedEnd = true; - res = SZ_ERROR_DATA; - } - } - else // status == CODER_STATUS_NOT_FINISHED - res = SZ_ERROR_DATA; - } - else if (res == SZ_ERROR_NO_ARCHIVE) - { - if (InSize == extraSize) - IsArc = false; - else - { - if (extraSize != 0 || inPos != inSize) - { - DataAfterEnd = true; - res = SZ_OK; - } - } - } - - DecodeRes = res; - PhySize -= extraSize; - - switch (res) - { - case SZ_OK: break; - case SZ_ERROR_NO_ARCHIVE: IsArc = false; break; - case SZ_ERROR_ARCHIVE: HeadersError = true; break; - case SZ_ERROR_UNSUPPORTED: Unsupported = true; break; - case SZ_ERROR_CRC: CrcError = true; break; - case SZ_ERROR_DATA: DataError = true; break; - default: DataError = true; break; - } - - return readRes; - } + if (outSizeLimit && *outSizeLimit != outWrap.Processed) + res = SZ_ERROR_DATA; } + + return SResToHRESULT_Code(res); } -Int32 CDecoder::Get_Extract_OperationResult() const -{ - Int32 opRes; - if (!IsArc) - opRes = NExtract::NOperationResult::kIsNotArc; - else if (UnexpectedEnd) - opRes = NExtract::NOperationResult::kUnexpectedEnd; - else if (DataAfterEnd) - opRes = NExtract::NOperationResult::kDataAfterEnd; - else if (CrcError) - opRes = NExtract::NOperationResult::kCRCError; - else if (Unsupported) - opRes = NExtract::NOperationResult::kUnsupportedMethod; - else if (HeadersError) - opRes = NExtract::NOperationResult::kDataError; - else if (DataError) - opRes = NExtract::NOperationResult::kDataError; - else if (DecodeRes != SZ_OK) - opRes = NExtract::NOperationResult::kDataError; - else - opRes = NExtract::NOperationResult::kOK; - return opRes; -} - - - HRESULT CComDecoder::Code(ISequentialInStream *inStream, ISequentialOutStream *outStream, const UInt64 * /* inSize */, const UInt64 *outSize, ICompressProgressInfo *progress) { - RINOK(_decoder.Decode(inStream, outStream, outSize, _finishStream, progress)); - Int32 opRes = _decoder.Get_Extract_OperationResult(); - if (opRes == NArchive::NExtract::NOperationResult::kUnsupportedMethod) - return E_NOTIMPL; - if (opRes != NArchive::NExtract::NOperationResult::kOK) - return S_FALSE; - return S_OK; + return Decode(inStream, outStream, outSize, _finishStream, progress); } STDMETHODIMP CComDecoder::SetFinishMode(UInt32 finishMode) @@ -256,8 +127,24 @@ STDMETHODIMP CComDecoder::SetFinishMode(UInt32 finishMode) STDMETHODIMP CComDecoder::GetInStreamProcessedSize(UInt64 *value) { - *value = _decoder.InSize; + *value = Stat.InSize; return S_OK; } +#ifndef _7ZIP_ST + +STDMETHODIMP CComDecoder::SetNumberOfThreads(UInt32 numThreads) +{ + _numThreads = numThreads; + return S_OK; +} + +STDMETHODIMP CComDecoder::SetMemLimit(UInt64 memUsage) +{ + _memUsage = memUsage; + return S_OK; +} + +#endif + }} diff --git a/CPP/7zip/Compress/XzDecoder.h b/CPP/7zip/Compress/XzDecoder.h index 9a3bf518..76694eec 100644 --- a/CPP/7zip/Compress/XzDecoder.h +++ b/CPP/7zip/Compress/XzDecoder.h @@ -12,57 +12,36 @@ namespace NCompress { namespace NXz { -struct CXzUnpackerCPP +struct CDecoder { - Byte *InBuf; - Byte *OutBuf; - CXzUnpacker p; + CXzDecMtHandle xz; + int _tryMt; + UInt32 _numThreads; + UInt64 _memUsage; + + SRes MainDecodeSRes; // it's not HRESULT + bool MainDecodeSRes_wasUsed; + CXzStatInfo Stat; + + CDecoder(): + xz(NULL), + _tryMt(True), + _numThreads(1), + _memUsage((UInt64)(sizeof(size_t)) << 28), + MainDecodeSRes(SZ_OK), + MainDecodeSRes_wasUsed(false) + {} - CXzUnpackerCPP(); - ~CXzUnpackerCPP(); -}; - - -struct CStatInfo -{ - UInt64 InSize; - UInt64 OutSize; - UInt64 PhySize; - - UInt64 NumStreams; - UInt64 NumBlocks; - - bool UnpackSize_Defined; - - bool NumStreams_Defined; - bool NumBlocks_Defined; - - bool IsArc; - bool UnexpectedEnd; - bool DataAfterEnd; - bool Unsupported; - bool HeadersError; - bool DataError; - bool CrcError; - - CStatInfo() { Clear(); } - - void Clear(); -}; - - -struct CDecoder: public CStatInfo -{ - CXzUnpackerCPP xzu; - SRes DecodeRes; // it's not HRESULT - - CDecoder(): DecodeRes(SZ_OK) {} + ~CDecoder() + { + if (xz) + XzDecMt_Destroy(xz); + } /* Decode() can return ERROR code only if there is progress or stream error. Decode() returns S_OK in case of xz decoding error, but DecodeRes and CStatInfo contain error information */ HRESULT Decode(ISequentialInStream *seqInStream, ISequentialOutStream *outStream, const UInt64 *outSizeLimit, bool finishStream, ICompressProgressInfo *compressProgress); - Int32 Get_Extract_OperationResult() const; }; @@ -70,21 +49,41 @@ class CComDecoder: public ICompressCoder, public ICompressSetFinishMode, public ICompressGetInStreamProcessedSize, - public CMyUnknownImp + + #ifndef _7ZIP_ST + public ICompressSetCoderMt, + public ICompressSetMemLimit, + #endif + + public CMyUnknownImp, + public CDecoder { - CDecoder _decoder; bool _finishStream; public: - MY_UNKNOWN_IMP2( - ICompressSetFinishMode, - ICompressGetInStreamProcessedSize) + MY_QUERYINTERFACE_BEGIN2(ICompressCoder) + + MY_QUERYINTERFACE_ENTRY(ICompressSetFinishMode) + MY_QUERYINTERFACE_ENTRY(ICompressGetInStreamProcessedSize) + + #ifndef _7ZIP_ST + MY_QUERYINTERFACE_ENTRY(ICompressSetCoderMt) + MY_QUERYINTERFACE_ENTRY(ICompressSetMemLimit) + #endif + MY_QUERYINTERFACE_END + MY_ADDREF_RELEASE + STDMETHOD(Code)(ISequentialInStream *inStream, ISequentialOutStream *outStream, const UInt64 *inSize, const UInt64 *outSize, ICompressProgressInfo *progress); STDMETHOD(SetFinishMode)(UInt32 finishMode); STDMETHOD(GetInStreamProcessedSize)(UInt64 *value); + #ifndef _7ZIP_ST + STDMETHOD(SetNumberOfThreads)(UInt32 numThreads); + STDMETHOD(SetMemLimit)(UInt64 memUsage); + #endif + CComDecoder(): _finishStream(false) {} }; diff --git a/CPP/7zip/Crypto/Rar5Aes.cpp b/CPP/7zip/Crypto/Rar5Aes.cpp index 45bf5f8b..c3384584 100644 --- a/CPP/7zip/Crypto/Rar5Aes.cpp +++ b/CPP/7zip/Crypto/Rar5Aes.cpp @@ -30,14 +30,13 @@ CDecoder::CDecoder(): CAesCbcDecoder(kAesKeySize) {} static unsigned ReadVarInt(const Byte *p, unsigned maxSize, UInt64 *val) { - unsigned i; *val = 0; - for (i = 0; i < maxSize;) + for (unsigned i = 0; i < maxSize && i < 10;) { Byte b = p[i]; - if (i < 10) - *val |= (UInt64)(b & 0x7F) << (7 * i++); + *val |= (UInt64)(b & 0x7F) << (7 * i); + i++; if ((b & 0x80) == 0) return i; } @@ -162,8 +161,9 @@ void CDecoder::Hmac_Convert_32Bytes(Byte *data) const }; +static CKey g_Key; + #ifndef _7ZIP_ST - static CKey g_Key; static NWindows::NSynchronization::CCriticalSection g_GlobalKeyCacheCriticalSection; #define MT_LOCK NWindows::NSynchronization::CCriticalSectionLock lock(g_GlobalKeyCacheCriticalSection); #else diff --git a/CPP/7zip/GuiCommon.rc b/CPP/7zip/GuiCommon.rc index c11017b1..565ee702 100644 --- a/CPP/7zip/GuiCommon.rc +++ b/CPP/7zip/GuiCommon.rc @@ -71,6 +71,9 @@ LANGUAGE LANG_ENGLISH, SUBLANG_ENGLISH_US DEFPUSHBUTTON "OK", IDOK, bx2, by, bxs, bys \ PUSHBUTTON "Cancel", IDCANCEL, bx1, by, bxs, bys +#define MY_BUTTON__CLOSE \ + DEFPUSHBUTTON "&Close", IDCLOSE, bx1, by, bxs, bys + #define MY_COMBO CBS_DROPDOWNLIST | WS_VSCROLL | WS_TABSTOP #define MY_COMBO_SORTED MY_COMBO | CBS_SORT diff --git a/CPP/7zip/Guid.txt b/CPP/7zip/Guid.txt index e896fe65..f9534942 100644 --- a/CPP/7zip/Guid.txt +++ b/CPP/7zip/Guid.txt @@ -49,6 +49,7 @@ 25 ICompressSetCoderMt 26 ICompressSetFinishMode 27 ICompressGetInStreamProcessedSize2 + 28 ICompressSetMemLimit 30 ICompressGetSubStreamSize 31 ICompressSetInStream @@ -169,7 +170,8 @@ Handler GUIDs: 10 lz5 11 liz - C6 Lzip + C5 Lzip + C6 COFF C7 Ext C8 VMDK C9 VDI diff --git a/CPP/7zip/ICoder.h b/CPP/7zip/ICoder.h index c0fe27e9..54b24765 100644 --- a/CPP/7zip/ICoder.h +++ b/CPP/7zip/ICoder.h @@ -43,6 +43,7 @@ CODER_INTERFACE(ICompressCoder2, 0x18) S_OK : OK S_FALSE : data error (for decoders) E_OUTOFMEMORY : memory allocation error + E_NOTIMPL : unsupported encoding method (for decoders) another error code : some error. For example, it can be error code received from inStream or outStream function. Parameters: @@ -129,7 +130,8 @@ namespace NCoderPropID kBlockSize2, // VT_UI4 or VT_UI8 kCheckSize, // VT_UI4 : size of digest in bytes - kFilter // VT_BSTR + kFilter, // VT_BSTR + kMemUse // VT_UI8 }; } @@ -190,6 +192,12 @@ CODER_INTERFACE(ICompressGetInStreamProcessedSize2, 0x27) STDMETHOD(GetInStreamProcessedSize2)(UInt32 streamIndex, UInt64 *value) PURE; }; +CODER_INTERFACE(ICompressSetMemLimit, 0x28) +{ + STDMETHOD(SetMemLimit)(UInt64 memUsage) PURE; +}; + + CODER_INTERFACE(ICompressGetSubStreamSize, 0x30) { diff --git a/CPP/7zip/LzmaDec.mak b/CPP/7zip/LzmaDec.mak new file mode 100644 index 00000000..3c0e7c5b --- /dev/null +++ b/CPP/7zip/LzmaDec.mak @@ -0,0 +1,5 @@ +!IF "$(CPU)" == "AMD64" +CFLAGS_C_SPEC = -D_LZMA_DEC_OPT +ASM_OBJS = $(ASM_OBJS) \ + $O\LzmaDecOpt.obj +!ENDIF diff --git a/CPP/7zip/UI/Agent/Agent.cpp b/CPP/7zip/UI/Agent/Agent.cpp index 32ef8026..730a9087 100644 --- a/CPP/7zip/UI/Agent/Agent.cpp +++ b/CPP/7zip/UI/Agent/Agent.cpp @@ -1217,7 +1217,12 @@ STDMETHODIMP CAgentFolder::GetFolderProperty(PROPID propID, PROPVARIANT *value) NWindows::NCOM::CPropVariant prop; if (propID == kpidReadOnly) - prop = _agentSpec->IsThereReadOnlyArc(); + { + if (_agentSpec->Is_Attrib_ReadOnly()) + prop = true; + else + prop = _agentSpec->IsThereReadOnlyArc(); + } else if (_proxy2) { const CProxyDir2 &dir = _proxy2->Dirs[_proxyDirIndex]; @@ -1507,11 +1512,18 @@ STDMETHODIMP CAgentFolder::Extract(const UInt32 *indices, #endif - HRESULT result = _agentSpec->GetArchive()->Extract(&realIndices.Front(), - realIndices.Size(), testMode, extractCallback); - if (result == S_OK) - result = extractCallbackSpec->SetDirsTimes(); - return result; + { + CArchiveExtractCallback_Closer ecsCloser(extractCallbackSpec); + + HRESULT res = _agentSpec->GetArchive()->Extract(&realIndices.Front(), + realIndices.Size(), testMode, extractCallback); + + HRESULT res2 = ecsCloser.Close(); + if (res == S_OK) + res = res2; + return res; + } + COM_TRY_END } @@ -1557,6 +1569,7 @@ STDMETHODIMP CAgent::Open( { COM_TRY_BEGIN _archiveFilePath = filePath; + _attrib = 0; NFile::NFind::CFileInfo fi; _isDeviceFile = false; if (!inStream) @@ -1565,6 +1578,7 @@ STDMETHODIMP CAgent::Open( return ::GetLastError(); if (fi.IsDir()) return E_FAIL; + _attrib = fi.Attrib; _isDeviceFile = fi.IsDevice; } CArcInfoEx archiverInfo0, archiverInfo1; diff --git a/CPP/7zip/UI/Agent/Agent.h b/CPP/7zip/UI/Agent/Agent.h index 688ea7af..2d5a239b 100644 --- a/CPP/7zip/UI/Agent/Agent.h +++ b/CPP/7zip/UI/Agent/Agent.h @@ -241,6 +241,7 @@ public: CAgentFolder *_agentFolder; UString _archiveFilePath; + DWORD _attrib; bool _isDeviceFile; #ifndef EXTRACT_ONLY @@ -252,6 +253,11 @@ public: IInArchive *GetArchive() const { if ( _archiveLink.Arcs.IsEmpty()) return 0; return GetArc().Archive; } bool CanUpdate() const; + bool Is_Attrib_ReadOnly() const + { + return _attrib != INVALID_FILE_ATTRIBUTES && (_attrib & FILE_ATTRIBUTE_READONLY); + } + bool IsThereReadOnlyArc() const { FOR_VECTOR (i, _archiveLink.Arcs) diff --git a/CPP/7zip/UI/Common/ArchiveCommandLine.cpp b/CPP/7zip/UI/Common/ArchiveCommandLine.cpp index bdf3ebcf..d5096325 100644 --- a/CPP/7zip/UI/Common/ArchiveCommandLine.cpp +++ b/CPP/7zip/UI/Common/ArchiveCommandLine.cpp @@ -8,9 +8,17 @@ #ifndef UNDER_CE #include #endif +#else +// for isatty() +#include #endif + #include +#ifdef _7ZIP_LARGE_PAGES +#include "../../../../C/Alloc.h" +#endif + #include "../../../Common/ListFileUtils.h" #include "../../../Common/StringConvert.h" #include "../../../Common/StringToInt.h" @@ -19,18 +27,20 @@ #include "../../../Windows/FileName.h" #ifdef _WIN32 #include "../../../Windows/FileMapping.h" +#include "../../../Windows/MemoryLock.h" #include "../../../Windows/Synchronization.h" #endif #include "ArchiveCommandLine.h" #include "EnumDirItems.h" -#include "SortUtils.h" #include "Update.h" #include "UpdateAction.h" extern bool g_CaseSensitive; extern bool g_PathTrailReplaceMode; +bool g_LargePagesMode = false; + #ifdef UNDER_CE #define MY_IS_TERMINAL(x) false; @@ -60,15 +70,6 @@ static bool StringToUInt32(const wchar_t *s, UInt32 &v) return *end == 0; } -CArcCmdLineException::CArcCmdLineException(const char *a, const wchar_t *u) -{ - (*this) += a; - if (u) - { - Add_LF(); - (*this) += u; - } -} int g_CodePage = -1; @@ -124,6 +125,7 @@ enum Enum kTechMode, kShareForWrite, + kStopAfterOpenError, kCaseSensitive, kArcNameMode, @@ -239,12 +241,13 @@ static const CSwitchForm kSwitchForms[] = { "si", NSwitchType::kString }, { "so" }, - { "slp", NSwitchType::kMinus }, + { "slp", NSwitchType::kString }, { "scs", NSwitchType::kString }, { "scc", NSwitchType::kString }, { "slt" }, { "ssw" }, + { "sse" }, { "ssc", NSwitchType::kMinus }, { "sa", NSwitchType::kChar, false, 1, k_ArcNameMode_PostCharSet }, @@ -280,7 +283,6 @@ static const char * const kIncorrectListFile = "Incorrect item in listfile.\nChe static const char * const kTerminalOutError = "I won't write compressed data to a terminal"; static const char * const kSameTerminalError = "I won't write data and program's messages to same stream"; static const char * const kEmptyFilePath = "Empty file path"; -static const char * const kCannotFindArchive = "Cannot find archive"; bool CArcCommand::IsFromExtractGroup() const { @@ -613,84 +615,6 @@ static void AddSwitchWildcardsToCensor( throw CArcCmdLineException(errorMessage, strings[i]); } -#ifdef _WIN32 - -// This code converts all short file names to long file names. - -static void ConvertToLongName(const UString &prefix, UString &name) -{ - if (name.IsEmpty() || DoesNameContainWildcard(name)) - return; - NFind::CFileInfo fi; - const FString path (us2fs(prefix + name)); - #ifndef UNDER_CE - if (NFile::NName::IsDevicePath(path)) - return; - #endif - if (fi.Find(path)) - name = fs2us(fi.Name); -} - -static void ConvertToLongNames(const UString &prefix, CObjectVector &items) -{ - FOR_VECTOR (i, items) - { - NWildcard::CItem &item = items[i]; - if (item.Recursive || item.PathParts.Size() != 1) - continue; - if (prefix.IsEmpty() && item.IsDriveItem()) - continue; - ConvertToLongName(prefix, item.PathParts.Front()); - } -} - -static void ConvertToLongNames(const UString &prefix, NWildcard::CCensorNode &node) -{ - ConvertToLongNames(prefix, node.IncludeItems); - ConvertToLongNames(prefix, node.ExcludeItems); - unsigned i; - for (i = 0; i < node.SubNodes.Size(); i++) - { - UString &name = node.SubNodes[i].Name; - if (prefix.IsEmpty() && NWildcard::IsDriveColonName(name)) - continue; - ConvertToLongName(prefix, name); - } - // mix folders with same name - for (i = 0; i < node.SubNodes.Size(); i++) - { - NWildcard::CCensorNode &nextNode1 = node.SubNodes[i]; - for (unsigned j = i + 1; j < node.SubNodes.Size();) - { - const NWildcard::CCensorNode &nextNode2 = node.SubNodes[j]; - if (nextNode1.Name.IsEqualTo_NoCase(nextNode2.Name)) - { - nextNode1.IncludeItems += nextNode2.IncludeItems; - nextNode1.ExcludeItems += nextNode2.ExcludeItems; - node.SubNodes.Delete(j); - } - else - j++; - } - } - for (i = 0; i < node.SubNodes.Size(); i++) - { - NWildcard::CCensorNode &nextNode = node.SubNodes[i]; - ConvertToLongNames(prefix + nextNode.Name + WCHAR_PATH_SEPARATOR, nextNode); - } -} - -void ConvertToLongNames(NWildcard::CCensor &censor) -{ - FOR_VECTOR (i, censor.Pairs) - { - NWildcard::CPair &pair = censor.Pairs[i]; - ConvertToLongNames(pair.Prefix, pair.Head); - } -} - -#endif - /* static NUpdateArchive::NPairAction::EEnum GetUpdatePairActionType(int i) { @@ -920,9 +844,45 @@ void CArcCmdLineParser::Parse1(const UStringVector &commandStrings, options.CaseSensitive = g_CaseSensitive; } - options.LargePages = false; + + #if defined(_WIN32) && !defined(UNDER_CE) + NSecurity::EnablePrivilege_SymLink(); + #endif + + // options.LargePages = false; + if (parser[NKey::kLargePages].ThereIs) - options.LargePages = !parser[NKey::kLargePages].WithMinus; + { + unsigned slp = 0; + const UString &s = parser[NKey::kLargePages].PostStrings[0]; + if (s.IsEmpty()) + slp = 1; + else if (s != L"-") + { + if (!StringToUInt32(s, slp)) + throw CArcCmdLineException("Unsupported switch postfix for -slp", s); + } + + #ifdef _7ZIP_LARGE_PAGES + if (slp > + #ifndef UNDER_CE + (unsigned)NSecurity::Get_LargePages_RiskLevel() + #else + 0 + #endif + ) + { + SetLargePageSize(); + // note: this process also can inherit that Privilege from parent process + g_LargePagesMode = + #if defined(_WIN32) && !defined(UNDER_CE) + NSecurity::EnablePrivilege_LockMemory(); + #else + true; + #endif + } + #endif + } #ifndef UNDER_CE @@ -994,64 +954,6 @@ static Int32 FindCharset(const NCommandLineParser::CParser &parser, unsigned key } } -HRESULT EnumerateDirItemsAndSort( - NWildcard::CCensor &censor, - NWildcard::ECensorPathMode censorPathMode, - const UString &addPathPrefix, - UStringVector &sortedPaths, - UStringVector &sortedFullPaths, - CDirItemsStat &st, - IDirItemsCallback *callback) -{ - FStringVector paths; - - { - CDirItems dirItems; - dirItems.Callback = callback; - { - HRESULT res = EnumerateItems(censor, censorPathMode, addPathPrefix, dirItems); - st = dirItems.Stat; - RINOK(res); - } - - FOR_VECTOR (i, dirItems.Items) - { - const CDirItem &dirItem = dirItems.Items[i]; - if (!dirItem.IsDir()) - paths.Add(dirItems.GetPhyPath(i)); - } - } - - if (paths.Size() == 0) - throw CArcCmdLineException(kCannotFindArchive); - - UStringVector fullPaths; - - unsigned i; - - for (i = 0; i < paths.Size(); i++) - { - FString fullPath; - NFile::NDir::MyGetFullPathName(paths[i], fullPath); - fullPaths.Add(fs2us(fullPath)); - } - - CUIntVector indices; - SortFileNames(fullPaths, indices); - sortedPaths.ClearAndReserve(indices.Size()); - sortedFullPaths.ClearAndReserve(indices.Size()); - - for (i = 0; i < indices.Size(); i++) - { - unsigned index = indices[i]; - sortedPaths.AddInReserved(fs2us(paths[index])); - sortedFullPaths.AddInReserved(fullPaths[index]); - if (i > 0 && CompareFileNames(sortedFullPaths[i], sortedFullPaths[i - 1]) == 0) - throw CArcCmdLineException("Duplicate archive path:", sortedFullPaths[i]); - } - - return S_OK; -} static void SetBoolPair(NCommandLineParser::CParser &parser, unsigned switchID, CBoolPair &bp) { @@ -1299,6 +1201,8 @@ void CArcCmdLineParser::Parse2(CArcCmdLineOptions &options) if (parser[NKey::kShareForWrite].ThereIs) updateOptions.OpenShareForWrite = true; + if (parser[NKey::kStopAfterOpenError].ThereIs) + updateOptions.StopAfterOpenError = true; updateOptions.PathMode = censorPathMode; diff --git a/CPP/7zip/UI/Common/ArchiveCommandLine.h b/CPP/7zip/UI/Common/ArchiveCommandLine.h index 97bf4f8d..bba3c98a 100644 --- a/CPP/7zip/UI/Common/ArchiveCommandLine.h +++ b/CPP/7zip/UI/Common/ArchiveCommandLine.h @@ -6,14 +6,13 @@ #include "../../../Common/CommandLineParser.h" #include "../../../Common/Wildcard.h" +#include "EnumDirItems.h" + #include "Extract.h" #include "HashCalc.h" #include "Update.h" -struct CArcCmdLineException: public UString -{ - CArcCmdLineException(const char *a, const wchar_t *u = NULL); -}; +typedef CMessagePathException CArcCmdLineException; namespace NCommandType { enum EEnum { @@ -51,7 +50,7 @@ struct CArcCmdLineOptions { bool HelpMode; - bool LargePages; + // bool LargePages; bool CaseSensitiveChange; bool CaseSensitive; @@ -110,7 +109,7 @@ struct CArcCmdLineOptions UInt32 NumIterations; CArcCmdLineOptions(): - LargePages(false), + // LargePages(false), CaseSensitiveChange(false), CaseSensitive(false), @@ -134,13 +133,4 @@ public: void Parse2(CArcCmdLineOptions &options); }; -HRESULT EnumerateDirItemsAndSort( - NWildcard::CCensor &censor, - NWildcard::ECensorPathMode pathMode, - const UString &addPathPrefix, - UStringVector &sortedPaths, - UStringVector &sortedFullPaths, - CDirItemsStat &st, - IDirItemsCallback *callback); - #endif diff --git a/CPP/7zip/UI/Common/ArchiveExtractCallback.cpp b/CPP/7zip/UI/Common/ArchiveExtractCallback.cpp index 5dd06d3c..1119d1b4 100644 --- a/CPP/7zip/UI/Common/ArchiveExtractCallback.cpp +++ b/CPP/7zip/UI/Common/ArchiveExtractCallback.cpp @@ -182,6 +182,7 @@ HRESULT CArchiveExtractCallback::PrepareHardLinks(const CRecordVector *r #endif CArchiveExtractCallback::CArchiveExtractCallback(): + _arc(NULL), WriteCTime(true), WriteATime(true), WriteMTime(true), @@ -205,8 +206,8 @@ void CArchiveExtractCallback::Init( const UStringVector &removePathParts, bool removePartsForAltStreams, UInt64 packSize) { - _extractedFolderPaths.Clear(); - _extractedFolderIndices.Clear(); + ClearExtractedDirsInfo(); + _outFileStream.Release(); #ifdef SUPPORT_LINKS _hardLinks.Clear(); @@ -368,9 +369,11 @@ void CArchiveExtractCallback::CreateComplexDirectory(const UStringVector &dirPat } } -HRESULT CArchiveExtractCallback::GetTime(int index, PROPID propID, FILETIME &filetime, bool &filetimeIsDefined) +HRESULT CArchiveExtractCallback::GetTime(UInt32 index, PROPID propID, FILETIME &filetime, bool &filetimeIsDefined) { filetimeIsDefined = false; + filetime.dwLowDateTime = 0; + filetime.dwHighDateTime = 0; NCOM::CPropVariant prop; RINOK(_arc->Archive->GetProperty(index, propID, &prop)); if (prop.vt == VT_FILETIME) @@ -734,7 +737,8 @@ STDMETHODIMP CArchiveExtractCallback::GetStream(UInt32 index, ISequentialOutStre return E_FAIL; UString s; CReparseAttr reparse; - isOkReparse = reparse.Parse((const Byte *)data, dataSize); + DWORD errorCode = 0; + isOkReparse = reparse.Parse((const Byte *)data, dataSize, errorCode); if (isOkReparse) { isHardLink = false; @@ -1030,14 +1034,36 @@ if (askExtractMode == NArchive::NExtract::NAskMode::kExtract && !_testMode) { FString fullPathNew; CreateComplexDirectory(pathParts, fullPathNew); + if (_item.IsDir) { - _extractedFolderPaths.Add(fullPathNew); - _extractedFolderIndices.Add(index); - SetDirTime(fullPathNew, - (WriteCTime && _fi.CTimeDefined) ? &_fi.CTime : NULL, - (WriteATime && _fi.ATimeDefined) ? &_fi.ATime : NULL, - (WriteMTime && _fi.MTimeDefined) ? &_fi.MTime : (_arc->MTimeDefined ? &_arc->MTime : NULL)); + CDirPathTime &pt = _extractedFolders.AddNew(); + + pt.CTime = _fi.CTime; + pt.CTimeDefined = (WriteCTime && _fi.CTimeDefined); + + pt.ATime = _fi.ATime; + pt.ATimeDefined = (WriteATime && _fi.ATimeDefined); + + pt.MTimeDefined = false; + + if (WriteMTime) + { + if (_fi.MTimeDefined) + { + pt.MTime = _fi.MTime; + pt.MTimeDefined = true; + } + else if (_arc->MTimeDefined) + { + pt.MTime = _arc->MTime; + pt.MTimeDefined = true; + } + } + + pt.Path = fullPathNew; + + pt.SetDirTime(); } } } @@ -1270,7 +1296,8 @@ if (askExtractMode == NArchive::NExtract::NAskMode::kExtract && !_testMode) if (FillLinkData(data, fs2us(existPath), !isJunction)) { CReparseAttr attr; - if (!attr.Parse(data, data.Size())) + DWORD errorCode = 0; + if (!attr.Parse(data, data.Size(), errorCode)) { RINOK(SendMessageError("Internal error for symbolic link file", us2fs(_item.Path))); // return E_FAIL; @@ -1447,6 +1474,33 @@ STDMETHODIMP CArchiveExtractCallback::PrepareOperation(Int32 askExtractMode) } +HRESULT CArchiveExtractCallback::CloseFile() +{ + if (!_outFileStream) + return S_OK; + + HRESULT hres = S_OK; + _outFileStreamSpec->SetTime( + (WriteCTime && _fi.CTimeDefined) ? &_fi.CTime : NULL, + (WriteATime && _fi.ATimeDefined) ? &_fi.ATime : NULL, + (WriteMTime && _fi.MTimeDefined) ? &_fi.MTime : (_arc->MTimeDefined ? &_arc->MTime : NULL)); + + const UInt64 processedSize = _outFileStreamSpec->ProcessedSize; + if (_fileLengthWasSet && _curSize > processedSize) + { + bool res = _outFileStreamSpec->File.SetLength(processedSize); + _fileLengthWasSet = res; + if (!res) + hres = SendMessageError_with_LastError(kCantSetFileLen, us2fs(_item.Path)); + } + _curSize = processedSize; + _curSizeDefined = true; + RINOK(_outFileStreamSpec->Close()); + _outFileStream.Release(); + return hres; +} + + STDMETHODIMP CArchiveExtractCallback::SetOperationResult(Int32 opRes) { COM_TRY_BEGIN @@ -1475,27 +1529,7 @@ STDMETHODIMP CArchiveExtractCallback::SetOperationResult(Int32 opRes) #endif - if (_outFileStream) - { - HRESULT hres = S_OK; - _outFileStreamSpec->SetTime( - (WriteCTime && _fi.CTimeDefined) ? &_fi.CTime : NULL, - (WriteATime && _fi.ATimeDefined) ? &_fi.ATime : NULL, - (WriteMTime && _fi.MTimeDefined) ? &_fi.MTime : (_arc->MTimeDefined ? &_arc->MTime : NULL)); - const UInt64 processedSize = _outFileStreamSpec->ProcessedSize; - if (_fileLengthWasSet && _curSize > processedSize) - { - bool res = _outFileStreamSpec->File.SetLength(processedSize); - _fileLengthWasSet = res; - if (!res) - hres = SendMessageError_with_LastError(kCantSetFileLen, us2fs(_item.Path)); - } - _curSize = processedSize; - _curSizeDefined = true; - RINOK(_outFileStreamSpec->Close()); - _outFileStream.Release(); - RINOK(hres); - } + RINOK(CloseFile()); #ifdef _USE_SECURITY_CODE if (!_stdOutMode && _extractMode && _ntOptions.NtSecurity.Val && _arc->GetRawProps) @@ -1592,71 +1626,66 @@ STDMETHODIMP CArchiveExtractCallback::CryptoGetTextPassword(BSTR *password) } -struct CExtrRefSortPair -{ - unsigned Len; - unsigned Index; - - int Compare(const CExtrRefSortPair &a) const; -}; - -#define RINOZ(x) { int __tt = (x); if (__tt != 0) return __tt; } - -int CExtrRefSortPair::Compare(const CExtrRefSortPair &a) const -{ - RINOZ(-MyCompare(Len, a.Len)); - return MyCompare(Index, a.Index); -} - -static unsigned GetNumSlashes(const FChar *s) +void CDirPathSortPair::SetNumSlashes(const FChar *s) { for (unsigned numSlashes = 0;;) { FChar c = *s++; if (c == 0) - return numSlashes; + { + Len = numSlashes; + return; + } if (IS_PATH_SEPAR(c)) numSlashes++; } } + +bool CDirPathTime::SetDirTime() +{ + return NDir::SetDirTime(Path, + CTimeDefined ? &CTime : NULL, + ATimeDefined ? &ATime : NULL, + MTimeDefined ? &MTime : NULL); +} + + HRESULT CArchiveExtractCallback::SetDirsTimes() { - CRecordVector pairs; - pairs.ClearAndSetSize(_extractedFolderPaths.Size()); + if (!_arc) + return S_OK; + + CRecordVector pairs; + pairs.ClearAndSetSize(_extractedFolders.Size()); unsigned i; - for (i = 0; i < _extractedFolderPaths.Size(); i++) + for (i = 0; i < _extractedFolders.Size(); i++) { - CExtrRefSortPair &pair = pairs[i]; + CDirPathSortPair &pair = pairs[i]; pair.Index = i; - pair.Len = GetNumSlashes(_extractedFolderPaths[i]); + pair.SetNumSlashes(_extractedFolders[i].Path); } pairs.Sort2(); for (i = 0; i < pairs.Size(); i++) { - int pairIndex = pairs[i].Index; - int index = _extractedFolderIndices[pairIndex]; - - FILETIME CTime; - FILETIME ATime; - FILETIME MTime; - - bool CTimeDefined; - bool ATimeDefined; - bool MTimeDefined; - - RINOK(GetTime(index, kpidCTime, CTime, CTimeDefined)); - RINOK(GetTime(index, kpidATime, ATime, ATimeDefined)); - RINOK(GetTime(index, kpidMTime, MTime, MTimeDefined)); - - // printf("\n%S", _extractedFolderPaths[pairIndex]); - SetDirTime(_extractedFolderPaths[pairIndex], - (WriteCTime && CTimeDefined) ? &CTime : NULL, - (WriteATime && ATimeDefined) ? &ATime : NULL, - (WriteMTime && MTimeDefined) ? &MTime : (_arc->MTimeDefined ? &_arc->MTime : NULL)); + _extractedFolders[pairs[i].Index].SetDirTime(); + // if (!) return GetLastError(); } + + ClearExtractedDirsInfo(); return S_OK; } + + +HRESULT CArchiveExtractCallback::CloseArc() +{ + HRESULT res = CloseFile(); + HRESULT res2 = SetDirsTimes(); + if (res == S_OK) + res = res2; + _arc = NULL; + return res; +} diff --git a/CPP/7zip/UI/Common/ArchiveExtractCallback.h b/CPP/7zip/UI/Common/ArchiveExtractCallback.h index ec857fa6..af38f13c 100644 --- a/CPP/7zip/UI/Common/ArchiveExtractCallback.h +++ b/CPP/7zip/UI/Common/ArchiveExtractCallback.h @@ -151,6 +151,25 @@ struct CIndexToPathPair #endif + + +struct CDirPathTime +{ + FILETIME CTime; + FILETIME ATime; + FILETIME MTime; + + bool CTimeDefined; + bool ATimeDefined; + bool MTimeDefined; + + FString Path; + + bool SetDirTime(); +}; + + + class CArchiveExtractCallback: public IArchiveExtractCallback, public IArchiveExtractCallbackMessage, @@ -241,15 +260,14 @@ class CArchiveExtractCallback: UInt64 _progressTotal; bool _progressTotal_Defined; - FStringVector _extractedFolderPaths; - CRecordVector _extractedFolderIndices; + CObjectVector _extractedFolders; #if defined(_WIN32) && !defined(UNDER_CE) && !defined(_SFX) bool _saclEnabled; #endif void CreateComplexDirectory(const UStringVector &dirPathParts, FString &fullPath); - HRESULT GetTime(int index, PROPID propID, FILETIME &filetime, bool &filetimeIsDefined); + HRESULT GetTime(UInt32 index, PROPID propID, FILETIME &filetime, bool &filetimeIsDefined); HRESULT GetUnpackSize(); HRESULT SendMessageError(const char *message, const FString &path); @@ -343,9 +361,43 @@ public: } #endif + HRESULT CloseArc(); + +private: + void ClearExtractedDirsInfo() + { + _extractedFolders.Clear(); + } + + HRESULT CloseFile(); HRESULT SetDirsTimes(); }; + +struct CArchiveExtractCallback_Closer +{ + CArchiveExtractCallback *_ref; + + CArchiveExtractCallback_Closer(CArchiveExtractCallback *ref): _ref(ref) {} + + HRESULT Close() + { + HRESULT res = S_OK; + if (_ref) + { + res = _ref->CloseArc(); + _ref = NULL; + } + return res; + } + + ~CArchiveExtractCallback_Closer() + { + Close(); + } +}; + + bool CensorNode_CheckPath(const NWildcard::CCensorNode &node, const CReadArcItem &item); #endif diff --git a/CPP/7zip/UI/Common/Bench.cpp b/CPP/7zip/UI/Common/Bench.cpp index f86e6e1e..c0d0e549 100644 --- a/CPP/7zip/UI/Common/Bench.cpp +++ b/CPP/7zip/UI/Common/Bench.cpp @@ -1183,9 +1183,11 @@ static HRESULT MethodBench( COneMethodInfo method = method2; UInt64 methodId; UInt32 numStreams; - if (!FindMethod( + int codecIndex = FindMethod_Index( EXTERNAL_CODECS_LOC_VARS - method.MethodName, methodId, numStreams)) + method.MethodName, true, + methodId, numStreams); + if (codecIndex < 0) return E_NOTIMPL; if (numStreams != 1) return E_INVALIDARG; @@ -1222,7 +1224,7 @@ static HRESULT MethodBench( { CCreatedCoder cod; - RINOK(CreateCoder(EXTERNAL_CODECS_LOC_VARS methodId, true, encoder._encoderFilter, cod)); + RINOK(CreateCoder_Index(EXTERNAL_CODECS_LOC_VARS codecIndex, true, encoder._encoderFilter, cod)); encoder._encoder = cod.Coder; if (!encoder._encoder && !encoder._encoderFilter) return E_NOTIMPL; @@ -1239,7 +1241,7 @@ static HRESULT MethodBench( { CCreatedCoder cod; CMyComPtr &decoder = encoder._decoders[j]; - RINOK(CreateCoder(EXTERNAL_CODECS_LOC_VARS methodId, false, encoder._decoderFilter, cod)); + RINOK(CreateCoder_Id(EXTERNAL_CODECS_LOC_VARS methodId, false, encoder._decoderFilter, cod)); decoder = cod.Coder; if (!encoder._decoderFilter && !decoder) return E_NOTIMPL; @@ -2600,7 +2602,8 @@ static void SysInfo_To_String(AString &s, const SYSTEM_INFO &si) PrintHex(s, si.wProcessorLevel); s += "."; PrintHex(s, si.wProcessorRevision); - if (si.dwActiveProcessorMask + 1 != ((UInt64)1 << si.dwNumberOfProcessors)) + if ((UInt64)si.dwActiveProcessorMask + 1 != ((UInt64)1 << si.dwNumberOfProcessors)) + if ((UInt64)si.dwActiveProcessorMask + 1 != 0 || si.dwNumberOfProcessors != sizeof(UInt64) * 8) { s += " act:"; PrintHex(s, si.dwActiveProcessorMask); @@ -2684,13 +2687,15 @@ void GetCpuName(AString &s) AString s2; x86cpuid_to_String(cpuid, s2); s += s2; - return; } + else + { #ifdef MY_CPU_AMD64 s += "x64"; #else s += "x86"; #endif + } } #else @@ -2701,6 +2706,9 @@ void GetCpuName(AString &s) #endif #endif + + if (g_LargePagesMode) + s += " (LP)"; } @@ -2723,6 +2731,27 @@ void GetCpuFeatures(AString &s) } +#ifdef _WIN32 +#ifndef UNDER_CE + +typedef void (WINAPI * Func_RtlGetVersion) (OSVERSIONINFOEXW *); + +static BOOL My_RtlGetVersion(OSVERSIONINFOEXW *vi) +{ + HMODULE ntdll = ::GetModuleHandleW(L"ntdll.dll"); + if (!ntdll) + return FALSE; + Func_RtlGetVersion func = (Func_RtlGetVersion)GetProcAddress(ntdll, "RtlGetVersion"); + if (!func) + return FALSE; + func(vi); + return TRUE; +} + +#endif +#endif + + HRESULT Bench( DECL_EXTERNAL_CODECS_LOC_VARS IBenchPrintCallback *printCallback, @@ -2857,6 +2886,30 @@ HRESULT Bench( if (printCallback) { + #ifdef _WIN32 + #ifndef UNDER_CE + { + AString s; + // OSVERSIONINFO vi; + OSVERSIONINFOEXW vi; + vi.dwOSVersionInfoSize = sizeof(vi); + // if (::GetVersionEx(&vi)) + if (My_RtlGetVersion(&vi)) + { + s += "Windows"; + if (vi.dwPlatformId != VER_PLATFORM_WIN32_NT) + s.Add_UInt32(vi.dwPlatformId); + s += " "; s.Add_UInt32(vi.dwMajorVersion); + s += "."; s.Add_UInt32(vi.dwMinorVersion); + s += " "; s.Add_UInt32(vi.dwBuildNumber); + // s += " "; s += GetAnsiString(vi.szCSDVersion); + } + printCallback->Print(s); + printCallback->NewLine(); + } + #endif + #endif + { AString s1, s2; GetSysInfo(s1, s2); diff --git a/CPP/7zip/UI/Common/CompressCall.cpp b/CPP/7zip/UI/Common/CompressCall.cpp index d00923b1..9fa46d7b 100644 --- a/CPP/7zip/UI/Common/CompressCall.cpp +++ b/CPP/7zip/UI/Common/CompressCall.cpp @@ -13,6 +13,7 @@ #include "../../../Windows/ErrorMsg.h" #include "../../../Windows/FileDir.h" #include "../../../Windows/FileMapping.h" +#include "../../../Windows/MemoryLock.h" #include "../../../Windows/ProcessUtils.h" #include "../../../Windows/Synchronization.h" @@ -96,6 +97,9 @@ static HRESULT Call7zGui(const UString ¶ms, static void AddLagePagesSwitch(UString ¶ms) { if (ReadLockMemoryEnable()) + #ifndef UNDER_CE + if (NSecurity::Get_LargePages_RiskLevel() == 0) + #endif params += " -slp"; } diff --git a/CPP/7zip/UI/Common/CompressCall2.cpp b/CPP/7zip/UI/Common/CompressCall2.cpp index 0a469673..df940fda 100644 --- a/CPP/7zip/UI/Common/CompressCall2.cpp +++ b/CPP/7zip/UI/Common/CompressCall2.cpp @@ -1,10 +1,10 @@ -// CompressCall.cpp +// CompressCall2.cpp #include "StdAfx.h" #include "../../../Common/MyException.h" -#include "../../UI/common/ArchiveCommandLine.h" +#include "../../UI/Common/EnumDirItems.h" #include "../../UI/GUI/BenchmarkDialog.h" #include "../../UI/GUI/ExtractGUI.h" @@ -20,6 +20,7 @@ extern HWND g_HWND; #define MY_TRY_BEGIN HRESULT result; try { #define MY_TRY_FINISH } \ catch(CSystemException &e) { result = e.ErrorCode; } \ + catch(UString &s) { ErrorMessage(s); result = E_FAIL; } \ catch(...) { result = E_FAIL; } \ if (result != S_OK && result != E_ABORT) \ ErrorMessageHRESULT(result); diff --git a/CPP/7zip/UI/Common/EnumDirItems.cpp b/CPP/7zip/UI/Common/EnumDirItems.cpp index 722bda0e..032e2fff 100644 --- a/CPP/7zip/UI/Common/EnumDirItems.cpp +++ b/CPP/7zip/UI/Common/EnumDirItems.cpp @@ -16,6 +16,7 @@ #endif #include "EnumDirItems.h" +#include "SortUtils.h" using namespace NWindows; using namespace NFile; @@ -386,15 +387,27 @@ HRESULT CDirItems::SetLinkInfo(CDirItem &dirItem, const NFind::CFileInfo &fi, return S_OK; const FString path = phyPrefix + fi.Name; CByteBuffer &buf = dirItem.ReparseData; + DWORD res = 0; if (NIO::GetReparseData(path, buf)) { CReparseAttr attr; - if (attr.Parse(buf, buf.Size())) + if (attr.Parse(buf, buf.Size(), res)) return S_OK; + // we ignore unknown reparse points + if (res != ERROR_INVALID_REPARSE_DATA) + res = 0; } - DWORD res = ::GetLastError(); + else + { + res = ::GetLastError(); + if (res == 0) + res = ERROR_INVALID_FUNCTION; + } + buf.Free(); - return AddError(path , res); + if (res == 0) + return S_OK; + return AddError(path, res); } #endif @@ -865,7 +878,8 @@ void CDirItems::FillFixedReparse() continue; CReparseAttr attr; - if (!attr.Parse(item.ReparseData, item.ReparseData.Size())) + DWORD errorCode = 0; + if (!attr.Parse(item.ReparseData, item.ReparseData.Size(), errorCode)) continue; if (attr.IsRelative()) continue; @@ -912,3 +926,161 @@ void CDirItems::FillFixedReparse() } #endif + + + +static const char * const kCannotFindArchive = "Cannot find archive"; + +HRESULT EnumerateDirItemsAndSort( + NWildcard::CCensor &censor, + NWildcard::ECensorPathMode censorPathMode, + const UString &addPathPrefix, + UStringVector &sortedPaths, + UStringVector &sortedFullPaths, + CDirItemsStat &st, + IDirItemsCallback *callback) +{ + FStringVector paths; + + { + CDirItems dirItems; + dirItems.Callback = callback; + { + HRESULT res = EnumerateItems(censor, censorPathMode, addPathPrefix, dirItems); + st = dirItems.Stat; + RINOK(res); + } + + FOR_VECTOR (i, dirItems.Items) + { + const CDirItem &dirItem = dirItems.Items[i]; + if (!dirItem.IsDir()) + paths.Add(dirItems.GetPhyPath(i)); + } + } + + if (paths.Size() == 0) + { + // return S_OK; + throw CMessagePathException(kCannotFindArchive); + } + + UStringVector fullPaths; + + unsigned i; + + for (i = 0; i < paths.Size(); i++) + { + FString fullPath; + NFile::NDir::MyGetFullPathName(paths[i], fullPath); + fullPaths.Add(fs2us(fullPath)); + } + + CUIntVector indices; + SortFileNames(fullPaths, indices); + sortedPaths.ClearAndReserve(indices.Size()); + sortedFullPaths.ClearAndReserve(indices.Size()); + + for (i = 0; i < indices.Size(); i++) + { + unsigned index = indices[i]; + sortedPaths.AddInReserved(fs2us(paths[index])); + sortedFullPaths.AddInReserved(fullPaths[index]); + if (i > 0 && CompareFileNames(sortedFullPaths[i], sortedFullPaths[i - 1]) == 0) + throw CMessagePathException("Duplicate archive path:", sortedFullPaths[i]); + } + + return S_OK; +} + + + + +#ifdef _WIN32 + +// This code converts all short file names to long file names. + +static void ConvertToLongName(const UString &prefix, UString &name) +{ + if (name.IsEmpty() || DoesNameContainWildcard(name)) + return; + NFind::CFileInfo fi; + const FString path (us2fs(prefix + name)); + #ifndef UNDER_CE + if (NFile::NName::IsDevicePath(path)) + return; + #endif + if (fi.Find(path)) + name = fs2us(fi.Name); +} + +static void ConvertToLongNames(const UString &prefix, CObjectVector &items) +{ + FOR_VECTOR (i, items) + { + NWildcard::CItem &item = items[i]; + if (item.Recursive || item.PathParts.Size() != 1) + continue; + if (prefix.IsEmpty() && item.IsDriveItem()) + continue; + ConvertToLongName(prefix, item.PathParts.Front()); + } +} + +static void ConvertToLongNames(const UString &prefix, NWildcard::CCensorNode &node) +{ + ConvertToLongNames(prefix, node.IncludeItems); + ConvertToLongNames(prefix, node.ExcludeItems); + unsigned i; + for (i = 0; i < node.SubNodes.Size(); i++) + { + UString &name = node.SubNodes[i].Name; + if (prefix.IsEmpty() && NWildcard::IsDriveColonName(name)) + continue; + ConvertToLongName(prefix, name); + } + // mix folders with same name + for (i = 0; i < node.SubNodes.Size(); i++) + { + NWildcard::CCensorNode &nextNode1 = node.SubNodes[i]; + for (unsigned j = i + 1; j < node.SubNodes.Size();) + { + const NWildcard::CCensorNode &nextNode2 = node.SubNodes[j]; + if (nextNode1.Name.IsEqualTo_NoCase(nextNode2.Name)) + { + nextNode1.IncludeItems += nextNode2.IncludeItems; + nextNode1.ExcludeItems += nextNode2.ExcludeItems; + node.SubNodes.Delete(j); + } + else + j++; + } + } + for (i = 0; i < node.SubNodes.Size(); i++) + { + NWildcard::CCensorNode &nextNode = node.SubNodes[i]; + ConvertToLongNames(prefix + nextNode.Name + WCHAR_PATH_SEPARATOR, nextNode); + } +} + +void ConvertToLongNames(NWildcard::CCensor &censor) +{ + FOR_VECTOR (i, censor.Pairs) + { + NWildcard::CPair &pair = censor.Pairs[i]; + ConvertToLongNames(pair.Prefix, pair.Head); + } +} + +#endif + + +CMessagePathException::CMessagePathException(const char *a, const wchar_t *u) +{ + (*this) += a; + if (u) + { + Add_LF(); + (*this) += u; + } +} diff --git a/CPP/7zip/UI/Common/EnumDirItems.h b/CPP/7zip/UI/Common/EnumDirItems.h index 15de3400..62205005 100644 --- a/CPP/7zip/UI/Common/EnumDirItems.h +++ b/CPP/7zip/UI/Common/EnumDirItems.h @@ -18,4 +18,24 @@ HRESULT EnumerateItems( const UString &addPathPrefix, CDirItems &dirItems); + +struct CMessagePathException: public UString +{ + CMessagePathException(const char *a, const wchar_t *u = NULL); +}; + + +HRESULT EnumerateDirItemsAndSort( + NWildcard::CCensor &censor, + NWildcard::ECensorPathMode pathMode, + const UString &addPathPrefix, + UStringVector &sortedPaths, + UStringVector &sortedFullPaths, + CDirItemsStat &st, + IDirItemsCallback *callback); + +#ifdef _WIN32 +void ConvertToLongNames(NWildcard::CCensor &censor); +#endif + #endif diff --git a/CPP/7zip/UI/Common/Extract.cpp b/CPP/7zip/UI/Common/Extract.cpp index ad9dff32..2cb2c9b9 100644 --- a/CPP/7zip/UI/Common/Extract.cpp +++ b/CPP/7zip/UI/Common/Extract.cpp @@ -197,6 +197,9 @@ static HRESULT DecompressArchive( HRESULT result; Int32 testMode = (options.TestMode && !calcCrc) ? 1: 0; + + CArchiveExtractCallback_Closer ecsCloser(ecs); + if (options.StdInMode) { result = archive->Extract(NULL, (UInt32)(Int32)-1, testMode, ecs); @@ -206,8 +209,11 @@ static HRESULT DecompressArchive( } else result = archive->Extract(&realIndices.Front(), realIndices.Size(), testMode, ecs); - if (result == S_OK && !options.StdInMode) - result = ecs->SetDirsTimes(); + + HRESULT res2 = ecsCloser.Close(); + if (result == S_OK) + result = res2; + return callback->ExtractResult(result); } diff --git a/CPP/7zip/UI/Common/HashCalc.h b/CPP/7zip/UI/Common/HashCalc.h index d98c5777..77373b85 100644 --- a/CPP/7zip/UI/Common/HashCalc.h +++ b/CPP/7zip/UI/Common/HashCalc.h @@ -9,7 +9,6 @@ #include "../../Common/MethodProps.h" #include "DirItem.h" -#include "Property.h" const unsigned k_HashCalc_DigestSize_Max = 64; diff --git a/CPP/7zip/UI/Common/OpenArchive.cpp b/CPP/7zip/UI/Common/OpenArchive.cpp index 6907239f..dcb81a64 100644 --- a/CPP/7zip/UI/Common/OpenArchive.cpp +++ b/CPP/7zip/UI/Common/OpenArchive.cpp @@ -857,7 +857,7 @@ HRESULT CArc::GetItem(UInt32 index, CReadArcItem &item) const if (item.WriteToAltStreamIfColon || needFindAltStream) { - /* Good handler must support GetRawProps::GetParent for alt streams./ + /* Good handler must support GetRawProps::GetParent for alt streams. So the following code currently is not used */ int colon = FindAltStreamColon_in_Path(item.Path); if (colon >= 0) @@ -992,7 +992,7 @@ static void MakeCheckOrder(CCodecs *codecs, int index = orderIndices[i]; if (index < 0) continue; - const CArcInfoEx &ai = codecs->Formats[index]; + const CArcInfoEx &ai = codecs->Formats[(unsigned)index]; if (ai.SignatureOffset != 0) { orderIndices2.Add(index); @@ -2296,7 +2296,7 @@ HRESULT CArc::OpenStream2(const COpenOptions &op) int index = orderIndices[i]; if (index < 0) continue; - const CArcInfoEx &ai = op.codecs->Formats[index]; + const CArcInfoEx &ai = op.codecs->Formats[(unsigned)index]; bool isDifficult = false; // if (ai.Version < 0x91F) // we don't use parser with old DLL (before 9.31) if (!ai.NewInterface) @@ -2328,7 +2328,7 @@ HRESULT CArc::OpenStream2(const COpenOptions &op) if (isDifficult) { difficultFormats.Add(index); - difficultBools[index] = true; + difficultBools[(unsigned)index] = true; } } diff --git a/CPP/7zip/UI/Common/OpenArchive.h b/CPP/7zip/UI/Common/OpenArchive.h index 03959871..033cb960 100644 --- a/CPP/7zip/UI/Common/OpenArchive.h +++ b/CPP/7zip/UI/Common/OpenArchive.h @@ -414,4 +414,23 @@ struct CArchiveLink bool ParseOpenTypes(CCodecs &codecs, const UString &s, CObjectVector &types); + +struct CDirPathSortPair +{ + unsigned Len; + unsigned Index; + + void SetNumSlashes(const FChar *s); + + int Compare(const CDirPathSortPair &a) const + { + // We need sorting order where parent items will be after child items + if (Len < a.Len) return 1; + if (Len > a.Len) return -1; + if (Index < a.Index) return -1; + if (Index > a.Index) return 1; + return 0; + } +}; + #endif diff --git a/CPP/7zip/UI/Common/PropIDUtils.cpp b/CPP/7zip/UI/Common/PropIDUtils.cpp index 8527da86..bb9c89f2 100644 --- a/CPP/7zip/UI/Common/PropIDUtils.cpp +++ b/CPP/7zip/UI/Common/PropIDUtils.cpp @@ -224,7 +224,6 @@ static inline void AddHexToString(AString &res, unsigned v) { res += (char)GetHex(v >> 4); res += (char)GetHex(v & 0xF); - res += ' '; } /* @@ -269,6 +268,14 @@ struct CSecID2Name const char *sz; }; +static int FindPairIndex(const CSecID2Name * pairs, unsigned num, UInt32 id) +{ + for (unsigned i = 0; i < num; i++) + if (pairs[i].n == id) + return i; + return -1; +} + static const CSecID2Name sid_32_Names[] = { { 544, "Administrators" }, @@ -359,22 +366,22 @@ static void ParseSid(AString &s, const Byte *p, UInt32 lim, UInt32 &sidSize) if (v0 == 32 && num == 2) { UInt32 v1 = Get32(p + 12); - for (unsigned i = 0; i < ARRAY_SIZE(sid_32_Names); i++) - if (sid_32_Names[i].n == v1) - { - s += sid_32_Names[i].sz; - return; - } + int index = FindPairIndex(sid_32_Names, ARRAY_SIZE(sid_32_Names), v1); + if (index >= 0) + { + s += sid_32_Names[(unsigned)index].sz; + return; + } } if (v0 == 21 && num == 5) { UInt32 v4 = Get32(p + 8 + 4 * 4); - for (unsigned i = 0; i < ARRAY_SIZE(sid_21_Names); i++) - if (sid_21_Names[i].n == v4) - { - s += sid_21_Names[i].sz; - return; - } + int index = FindPairIndex(sid_21_Names, ARRAY_SIZE(sid_21_Names), v4); + if (index >= 0) + { + s += sid_21_Names[(unsigned)index].sz; + return; + } } if (v0 == 80 && num == 6) { @@ -419,20 +426,13 @@ static void ParseOwner(AString &s, const Byte *p, UInt32 size, UInt32 pos) ParseSid(s, p + pos, size - pos, sidSize); } -static void AddUInt32ToString(AString &s, UInt32 val) -{ - char sz[16]; - ConvertUInt32ToString(val, sz); - s += sz; -} - static void ParseAcl(AString &s, const Byte *p, UInt32 size, const char *strName, UInt32 flags, UInt32 offset) { UInt32 control = Get16(p + 2); if ((flags & control) == 0) return; UInt32 pos = Get32(p + offset); - s += ' '; + s.Add_Space(); s += strName; if (pos >= size) return; @@ -443,7 +443,7 @@ static void ParseAcl(AString &s, const Byte *p, UInt32 size, const char *strName if (Get16(p) != 2) // revision return; UInt32 num = Get32(p + 4); - AddUInt32ToString(s, num); + s.Add_UInt32(num); /* UInt32 aclSize = Get16(p + 2); @@ -466,7 +466,7 @@ static void ParseAcl(AString &s, const Byte *p, UInt32 size, const char *strName size -= 8; UInt32 sidSize = 0; - s += ' '; + s.Add_Space(); ParseSid(s, p, size, sidSize); if (sidSize == 0) return; @@ -508,12 +508,12 @@ void ConvertNtSecureToString(const Byte *data, UInt32 size, AString &s) return; } ParseOwner(s, data, size, Get32(data + 4)); - s += ' '; + s.Add_Space(); ParseOwner(s, data, size, Get32(data + 8)); ParseAcl(s, data, size, "s:", MY_SE_SACL_PRESENT, 12); ParseAcl(s, data, size, "d:", MY_SE_DACL_PRESENT, 16); - s += ' '; - AddUInt32ToString(s, size); + s.Add_Space(); + s.Add_UInt32(size); // s += '\n'; // s += Data_To_Hex(data, size); } @@ -567,11 +567,38 @@ bool CheckNtSecure(const Byte *data, UInt32 size) throw() #endif + + +// IO_REPARSE_TAG_* + +static const CSecID2Name k_ReparseTags[] = +{ + { 0xA0000003, "MOUNT_POINT" }, + { 0xC0000004, "HSM" }, + { 0x80000005, "DRIVE_EXTENDER" }, + { 0x80000006, "HSM2" }, + { 0x80000007, "SIS" }, + { 0x80000008, "WIM" }, + { 0x80000009, "CSV" }, + { 0x8000000A, "DFS" }, + { 0x8000000B, "FILTER_MANAGER" }, + { 0xA000000C, "SYMLINK" }, + { 0xA0000010, "IIS_CACHE" }, + { 0x80000012, "DFSR" }, + { 0x80000013, "DEDUP" }, + { 0xC0000014, "APPXSTRM" }, + { 0x80000014, "NFS" }, + { 0x80000015, "FILE_PLACEHOLDER" }, + { 0x80000016, "DFM" }, + { 0x80000017, "WOF" } +}; + bool ConvertNtReparseToString(const Byte *data, UInt32 size, UString &s) { s.Empty(); NFile::CReparseAttr attr; - if (attr.Parse(data, size)) + DWORD errorCode = 0; + if (attr.Parse(data, size, errorCode)) { if (!attr.IsSymLink()) s += "Junction: "; @@ -593,19 +620,48 @@ bool ConvertNtReparseToString(const Byte *data, UInt32 size, UString &s) if (Get16(data + 6) != 0) // padding return false; - char hex[16]; - ConvertUInt32ToHex8Digits(tag, hex); - s += hex; - s.Add_Space(); - - data += 8; - - for (UInt32 i = 0; i < len; i++) + /* + #define _my_IO_REPARSE_TAG_DEDUP (0x80000013L) + if (tag == _my_IO_REPARSE_TAG_DEDUP) { - unsigned b = ((const Byte *)data)[i]; - s += (char)GetHex((b >> 4) & 0xF); - s += (char)GetHex(b & 0xF); } + */ + + { + int index = FindPairIndex(k_ReparseTags, ARRAY_SIZE(k_ReparseTags), tag); + if (index >= 0) + s += k_ReparseTags[(unsigned)index].sz; + else + { + s += "REPARSE:"; + char hex[16]; + ConvertUInt32ToHex8Digits(tag, hex); + s += hex; + } + } + + s += ":"; + s.Add_UInt32(len); + + if (len != 0) + { + s.Add_Space(); + + data += 8; + + for (UInt32 i = 0; i < len; i++) + { + if (i >= 8) + { + s += "..."; + break; + } + unsigned b = data[i]; + s += (char)GetHex((b >> 4) & 0xF); + s += (char)GetHex(b & 0xF); + } + } + return true; } diff --git a/CPP/7zip/UI/Common/Update.cpp b/CPP/7zip/UI/Common/Update.cpp index 76800f8f..fc1eede4 100644 --- a/CPP/7zip/UI/Common/Update.cpp +++ b/CPP/7zip/UI/Common/Update.cpp @@ -735,6 +735,7 @@ static HRESULT Compress( CMyComPtr updateCallback(updateCallbackSpec); updateCallbackSpec->ShareForWrite = options.OpenShareForWrite; + updateCallbackSpec->StopAfterOpenError = options.StopAfterOpenError; updateCallbackSpec->StdInMode = options.StdInMode; updateCallbackSpec->Callback = callback; @@ -1045,36 +1046,6 @@ static HRESULT EnumerateInArchiveItems( #endif -struct CRefSortPair -{ - unsigned Len; - unsigned Index; -}; - -#define RINOZ(x) { int __tt = (x); if (__tt != 0) return __tt; } - -static int CompareRefSortPair(const CRefSortPair *a1, const CRefSortPair *a2, void *) -{ - RINOZ(-MyCompare(a1->Len, a2->Len)); - return MyCompare(a1->Index, a2->Index); -} - -static unsigned GetNumSlashes(const FChar *s) -{ - for (unsigned numSlashes = 0;;) - { - FChar c = *s++; - if (c == 0) - return numSlashes; - if (IS_PATH_SEPAR(c)) - numSlashes++; - } -} - -#ifdef _WIN32 -void ConvertToLongNames(NWildcard::CCensor &censor); -#endif - HRESULT UpdateArchive( CCodecs *codecs, const CObjectVector &types, @@ -1189,6 +1160,16 @@ HRESULT UpdateArchive( throw "there is no such archive"; if (fi.IsDevice) return E_NOTIMPL; + + if (!options.StdOutMode && options.UpdateArchiveItself) + if (fi.IsReadOnly()) + { + errorInfo.SystemError = ERROR_ACCESS_DENIED; + errorInfo.Message = "The file is read-only"; + errorInfo.FileNames.Add(arcPath); + return errorInfo.Get_HRESULT_Error(); + } + if (options.VolumesSizes.Size() > 0) { errorInfo.FileNames.Add(us2fs(arcPath)); @@ -1509,9 +1490,13 @@ HRESULT UpdateArchive( CArchivePath &ap = options.Commands[0].ArchivePath; const FString &tempPath = ap.GetTempPath(); + // DWORD attrib = 0; if (thereIsInArchive) + { + // attrib = NFind::GetFileAttrib(us2fs(arcPath)); if (!DeleteFileAlways(us2fs(arcPath))) return errorInfo.SetFromLastError("cannot delete the file", us2fs(arcPath)); + } if (!MyMoveFile(tempPath, us2fs(arcPath))) { @@ -1519,6 +1504,15 @@ HRESULT UpdateArchive( errorInfo.FileNames.Add(us2fs(arcPath)); return errorInfo.Get_HRESULT_Error(); } + + /* + if (attrib != INVALID_FILE_ATTRIBUTES && (attrib & FILE_ATTRIBUTE_READONLY)) + { + DWORD attrib2 = NFind::GetFileAttrib(us2fs(arcPath)); + if (attrib2 != INVALID_FILE_ATTRIBUTES) + NDir::SetFileAttrib(us2fs(arcPath), attrib2 | FILE_ATTRIBUTE_READONLY); + } + */ } catch(...) { @@ -1608,7 +1602,7 @@ HRESULT UpdateArchive( if (options.DeleteAfterCompressing) { - CRecordVector pairs; + CRecordVector pairs; FStringVector foldersNames; unsigned i; @@ -1616,12 +1610,12 @@ HRESULT UpdateArchive( for (i = 0; i < dirItems.Items.Size(); i++) { const CDirItem &dirItem = dirItems.Items[i]; - FString phyPath = dirItems.GetPhyPath(i); + const FString phyPath = dirItems.GetPhyPath(i); if (dirItem.IsDir()) { - CRefSortPair pair; + CDirPathSortPair pair; pair.Index = i; - pair.Len = GetNumSlashes(phyPath); + pair.SetNumSlashes(phyPath); pairs.Add(pair); } else @@ -1654,11 +1648,11 @@ HRESULT UpdateArchive( } } - pairs.Sort(CompareRefSortPair, NULL); + pairs.Sort2(); for (i = 0; i < pairs.Size(); i++) { - FString phyPath = dirItems.GetPhyPath(pairs[i].Index); + const FString phyPath = dirItems.GetPhyPath(pairs[i].Index); if (NFind::DoesDirExist(phyPath)) { RINOK(callback->DeletingAfterArchiving(phyPath, true)); diff --git a/CPP/7zip/UI/Common/Update.h b/CPP/7zip/UI/Common/Update.h index 4e3fb2a8..45b02d74 100644 --- a/CPP/7zip/UI/Common/Update.h +++ b/CPP/7zip/UI/Common/Update.h @@ -92,6 +92,7 @@ struct CUpdateOptions FString SfxModule; bool OpenShareForWrite; + bool StopAfterOpenError; bool StdInMode; UString StdInFileName; @@ -127,6 +128,7 @@ struct CUpdateOptions EMailMode(false), EMailRemoveAfter(false), OpenShareForWrite(false), + StopAfterOpenError(false), ArcNameMode(k_ArcNameMode_Smart), PathMode(NWildcard::k_RelatPath), diff --git a/CPP/7zip/UI/Common/UpdateCallback.cpp b/CPP/7zip/UI/Common/UpdateCallback.cpp index b38f0056..9c165fe9 100644 --- a/CPP/7zip/UI/Common/UpdateCallback.cpp +++ b/CPP/7zip/UI/Common/UpdateCallback.cpp @@ -55,6 +55,7 @@ CArchiveUpdateCallback::CArchiveUpdateCallback(): Comment(NULL), ShareForWrite(false), + StopAfterOpenError(false), StdInMode(false), KeepOriginalItemNames(false), @@ -346,7 +347,8 @@ STDMETHODIMP CArchiveUpdateCallback::GetProperty(UInt32 index, PROPID propID, PR // if (di.IsDir()) { CReparseAttr attr; - if (attr.Parse(di.ReparseData, di.ReparseData.Size())) + DWORD errorCode = 0; + if (attr.Parse(di.ReparseData, di.ReparseData.Size(), errorCode)) { UString simpleName = attr.GetPath(); if (attr.IsRelative()) @@ -512,7 +514,12 @@ STDMETHODIMP CArchiveUpdateCallback::GetStream2(UInt32 index, ISequentialInStrea #endif if (!inStreamSpec->OpenShared(path, ShareForWrite)) { - return Callback->OpenFileError(path, ::GetLastError()); + DWORD error = ::GetLastError(); + HRESULT hres = Callback->OpenFileError(path, error); + if (StopAfterOpenError) + if (hres == S_OK || hres == S_FALSE) + return HRESULT_FROM_WIN32(error); + return hres; } if (StoreHardLinks) diff --git a/CPP/7zip/UI/Common/UpdateCallback.h b/CPP/7zip/UI/Common/UpdateCallback.h index 1cc35328..1b5d1eee 100644 --- a/CPP/7zip/UI/Common/UpdateCallback.h +++ b/CPP/7zip/UI/Common/UpdateCallback.h @@ -136,6 +136,7 @@ public: const UString *Comment; bool ShareForWrite; + bool StopAfterOpenError; bool StdInMode; bool KeepOriginalItemNames; diff --git a/CPP/7zip/UI/Console/Console.manifest b/CPP/7zip/UI/Console/Console.manifest new file mode 100644 index 00000000..77ecaad7 --- /dev/null +++ b/CPP/7zip/UI/Console/Console.manifest @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/CPP/7zip/UI/Console/ExtractCallbackConsole.cpp b/CPP/7zip/UI/Console/ExtractCallbackConsole.cpp index c8d83221..bdf95499 100644 --- a/CPP/7zip/UI/Console/ExtractCallbackConsole.cpp +++ b/CPP/7zip/UI/Console/ExtractCallbackConsole.cpp @@ -60,8 +60,9 @@ HRESULT CExtractScanConsole::ScanError(const FString &path, DWORD systemError) if (_se) { - *_se << endl << kError << NError::MyFormatMessage(systemError) << endl << - fs2us(path) << endl << endl; + *_se << endl << kError << NError::MyFormatMessage(systemError) << endl; + _se->NormalizePrint_UString(fs2us(path)); + *_se << endl << endl; _se->Flush(); } return HRESULT_FROM_WIN32(systemError); @@ -95,6 +96,16 @@ void PrintSize_bytes_Smart(AString &s, UInt64 val) s += ')'; } +void PrintSize_bytes_Smart_comma(AString &s, UInt64 val) +{ + if (val == (UInt64)(Int64)-1) + return; + s += ", "; + PrintSize_bytes_Smart(s, val); +} + + + void Print_DirItemsStat(AString &s, const CDirItemsStat &st) { if (st.NumDirs != 0) @@ -103,14 +114,12 @@ void Print_DirItemsStat(AString &s, const CDirItemsStat &st) s += ", "; } Print_UInt64_and_String(s, st.NumFiles, st.NumFiles == 1 ? "file" : "files"); - s += ", "; - PrintSize_bytes_Smart(s, st.FilesSize); + PrintSize_bytes_Smart_comma(s, st.FilesSize); if (st.NumAltStreams != 0) { s.Add_LF(); Print_UInt64_and_String(s, st.NumAltStreams, "alternate streams"); - s += ", "; - PrintSize_bytes_Smart(s, st.AltStreamsSize); + PrintSize_bytes_Smart_comma(s, st.AltStreamsSize); } } @@ -243,8 +252,10 @@ static const char * const kTab = " "; static void PrintFileInfo(CStdOutStream *_so, const wchar_t *path, const FILETIME *ft, const UInt64 *size) { - *_so << kTab << "Path: " << path << endl; - if (size) + *_so << kTab << "Path: "; + _so->NormalizePrint_wstr(path); + *_so << endl; + if (size && *size != (UInt64)(Int64)-1) { AString s; PrintSize_bytes_Smart(s, *size); @@ -332,7 +343,10 @@ STDMETHODIMP CExtractCallbackConsole::PrepareOperation(const wchar_t *name, Int3 _tempU.Empty(); if (name) + { _tempU = name; + _so->Normalize_UString(_tempU); + } _so->PrintUString(_tempU, _tempA); if (position) *_so << " <" << *position << ">"; @@ -453,7 +467,10 @@ STDMETHODIMP CExtractCallbackConsole::SetOperationResult(Int32 opRes, Int32 encr *_se << s; if (!_currentName.IsEmpty()) - *_se << " : " << _currentName; + { + *_se << " : "; + _se->NormalizePrint_UString(_currentName); + } *_se << endl; _se->Flush(); } @@ -505,7 +522,11 @@ HRESULT CExtractCallbackConsole::BeforeOpen(const wchar_t *name, bool testMode) ClosePercents_for_so(); if (_so) - *_so << endl << (testMode ? kTesting : kExtracting) << name << endl; + { + *_so << endl << (testMode ? kTesting : kExtracting); + _so->NormalizePrint_wstr(name); + *_so << endl; + } if (NeedPercents()) _percent.Command = "Open"; @@ -565,8 +586,9 @@ void Print_ErrorFormatIndex_Warning(CStdOutStream *_so, const CCodecs *codecs, c { const CArcErrorInfo &er = arc.ErrorInfo; - UString s ("WARNING:\n"); - s += arc.Path; + *_so << "WARNING:\n"; + _so->NormalizePrint_UString(arc.Path); + UString s; if (arc.FormatIndex == er.ErrorFormatIndex) { s.Add_LF(); @@ -611,7 +633,10 @@ HRESULT CExtractCallbackConsole::OpenResult( { *_se << endl; if (level != 0) - *_se << arc.Path << endl; + { + _se->NormalizePrint_UString(arc.Path); + *_se << endl; + } } if (errorFlags != 0) @@ -645,7 +670,10 @@ HRESULT CExtractCallbackConsole::OpenResult( { *_so << endl; if (level != 0) - *_so << arc.Path << endl; + { + _so->NormalizePrint_UString(arc.Path); + *_so << endl; + } } if (warningFlags != 0) @@ -700,7 +728,9 @@ HRESULT CExtractCallbackConsole::OpenResult( _so->Flush(); if (_se) { - *_se << kError << name << endl; + *_se << kError; + _se->NormalizePrint_wstr(name); + *_se << endl; HRESULT res = Print_OpenArchive_Error(*_se, codecs, arcLink); RINOK(res); if (result == S_FALSE) diff --git a/CPP/7zip/UI/Console/HashCon.cpp b/CPP/7zip/UI/Console/HashCon.cpp index 0f072b3b..ec8e6dca 100644 --- a/CPP/7zip/UI/Console/HashCon.cpp +++ b/CPP/7zip/UI/Console/HashCon.cpp @@ -269,7 +269,7 @@ HRESULT CHashCallbackConsole::SetOperationResult(UInt64 fileSize, const CHashBun if (_fileName.IsEmpty()) *_so << kEmptyFileAlias; else - *_so << _fileName; + _so->NormalizePrint_UString(_fileName); } *_so << endl; } diff --git a/CPP/7zip/UI/Console/List.cpp b/CPP/7zip/UI/Console/List.cpp index a6e7f87e..ebcabb68 100644 --- a/CPP/7zip/UI/Console/List.cpp +++ b/CPP/7zip/UI/Console/List.cpp @@ -560,7 +560,7 @@ HRESULT CFieldPrinter::PrintItemInfo(UInt32 index, const CListStat &st) { if (!techMode) g_StdOut << temp; - g_StdOut.PrintUString(FilePath, TempAString); + g_StdOut.NormalizePrint_UString(FilePath, TempWString, TempAString); if (techMode) g_StdOut << MY_ENDL; continue; @@ -671,9 +671,10 @@ HRESULT CFieldPrinter::PrintItemInfo(UInt32 index, const CListStat &st) else if (prop.vt == VT_BSTR) { TempWString.SetFromBstr(prop.bstrVal); + // do we need multi-line support here ? + g_StdOut.Normalize_UString(TempWString); if (techMode) { - // replace CR/LF here. g_StdOut.PrintUString(TempWString, TempAString); } else @@ -815,9 +816,63 @@ static void PrintPropNameAndNumber_Signed(CStdOutStream &so, PROPID propID, Int6 so << val << endl; } -static void PrintPropPair(CStdOutStream &so, const char *name, const wchar_t *val) + +static void UString_Replace_CRLF_to_LF(UString &s) { - so << name << " = " << val << endl; + // s.Replace(L"\r\n", L"\n"); + wchar_t *src = s.GetBuf(); + wchar_t *dest = src; + for (;;) + { + wchar_t c = *src++; + if (c == 0) + break; + if (c == '\r' && *src == '\n') + { + src++; + c = '\n'; + } + *dest++ = c; + } + s.ReleaseBuf_SetEnd((unsigned)(dest - s.GetBuf())); +} + + +static void PrintPropVal_MultiLine(CStdOutStream &so, const wchar_t *val) +{ + UString s = val; + if (s.Find(L'\n') >= 0) + { + so << endl; + so << "{"; + so << endl; + UString_Replace_CRLF_to_LF(s); + so.Normalize_UString__LF_Allowed(s); + so << s; + so << endl; + so << "}"; + } + else + { + so.Normalize_UString(s); + so << s; + } + so << endl; +} + + +static void PrintPropPair(CStdOutStream &so, const char *name, const wchar_t *val, bool multiLine) +{ + so << name << " = "; + if (multiLine) + { + PrintPropVal_MultiLine(so, val); + return; + } + UString s = val; + so.Normalize_UString(s); + so << s; + so << endl; } @@ -831,9 +886,11 @@ static void PrintPropertyPair2(CStdOutStream &so, PROPID propID, const wchar_t * UString nameU; GetPropName(propID, name, nameA, nameU); if (!nameA.IsEmpty()) - PrintPropPair(so, nameA, s); + so << nameA; else - so << nameU << " = " << s << endl; + so << nameU; + so << " = "; + PrintPropVal_MultiLine(so, s); } } @@ -862,11 +919,11 @@ static void ErrorInfo_Print(CStdOutStream &so, const CArcErrorInfo &er) { PrintErrorFlags(so, "ERRORS:", er.GetErrorFlags()); if (!er.ErrorMessage.IsEmpty()) - PrintPropPair(so, "ERROR", er.ErrorMessage); + PrintPropPair(so, "ERROR", er.ErrorMessage, true); PrintErrorFlags(so, "WARNINGS:", er.GetWarningFlags()); if (!er.WarningMessage.IsEmpty()) - PrintPropPair(so, "WARNING", er.WarningMessage); + PrintPropPair(so, "WARNING", er.WarningMessage, true); } HRESULT Print_OpenArchive_Props(CStdOutStream &so, const CCodecs *codecs, const CArchiveLink &arcLink) @@ -877,7 +934,7 @@ HRESULT Print_OpenArchive_Props(CStdOutStream &so, const CCodecs *codecs, const const CArcErrorInfo &er = arc.ErrorInfo; so << "--\n"; - PrintPropPair(so, "Path", arc.Path); + PrintPropPair(so, "Path", arc.Path, false); if (er.ErrorFormatIndex >= 0) { if (er.ErrorFormatIndex == arc.FormatIndex) @@ -885,7 +942,7 @@ HRESULT Print_OpenArchive_Props(CStdOutStream &so, const CCodecs *codecs, const else PrintArcTypeError(so, codecs->GetFormatNamePtr(er.ErrorFormatIndex), true); } - PrintPropPair(so, "Type", codecs->GetFormatNamePtr(arc.FormatIndex)); + PrintPropPair(so, "Type", codecs->GetFormatNamePtr(arc.FormatIndex), false); ErrorInfo_Print(so, er); @@ -943,7 +1000,8 @@ HRESULT Print_OpenArchive_Error(CStdOutStream &so, const CCodecs *codecs, const { if (arcLink.NonOpen_ErrorInfo.ErrorFormatIndex >= 0) { - so << arcLink.NonOpen_ArcPath << endl; + so.NormalizePrint_UString(arcLink.NonOpen_ArcPath); + so << endl; PrintArcTypeError(so, codecs->Formats[arcLink.NonOpen_ErrorInfo.ErrorFormatIndex].Name, false); } else @@ -1014,15 +1072,18 @@ HRESULT ListArchives(CCodecs *codecs, errorCode = ERROR_FILE_NOT_FOUND; lastError = HRESULT_FROM_WIN32(lastError);; g_StdOut.Flush(); - *g_ErrStream << endl << kError << NError::MyFormatMessage(errorCode) << - endl << arcPath << endl << endl; + *g_ErrStream << endl << kError << NError::MyFormatMessage(errorCode) << endl; + g_ErrStream->NormalizePrint_UString(arcPath); + *g_ErrStream << endl << endl; numErrors++; continue; } if (fi.IsDir()) { g_StdOut.Flush(); - *g_ErrStream << endl << kError << arcPath << " is not a file" << endl << endl; + *g_ErrStream << endl << kError; + g_ErrStream->NormalizePrint_UString(arcPath); + *g_ErrStream << " is not a file" << endl << endl; numErrors++; continue; } @@ -1061,7 +1122,9 @@ HRESULT ListArchives(CCodecs *codecs, if (enableHeaders) { - g_StdOut << endl << kListing << arcPath << endl << endl; + g_StdOut << endl << kListing; + g_StdOut.NormalizePrint_UString(arcPath); + g_StdOut << endl << endl; } HRESULT result = arcLink.Open_Strict(options, &openCallback); @@ -1071,7 +1134,9 @@ HRESULT ListArchives(CCodecs *codecs, if (result == E_ABORT) return result; g_StdOut.Flush(); - *g_ErrStream << endl << kError << arcPath << " : "; + *g_ErrStream << endl << kError; + g_ErrStream->NormalizePrint_UString(arcPath); + *g_ErrStream << " : "; if (result == S_FALSE) { Print_OpenArchive_Error(*g_ErrStream, codecs, arcLink); @@ -1255,7 +1320,7 @@ HRESULT ListArchives(CCodecs *codecs, if (arcLink.NonOpen_ErrorInfo.ErrorFormatIndex >= 0) { g_StdOut << "----------\n"; - PrintPropPair(g_StdOut, "Path", arcLink.NonOpen_ArcPath); + PrintPropPair(g_StdOut, "Path", arcLink.NonOpen_ArcPath, false); PrintArcTypeError(g_StdOut, codecs->Formats[arcLink.NonOpen_ErrorInfo.ErrorFormatIndex].Name, false); } } diff --git a/CPP/7zip/UI/Console/Main.cpp b/CPP/7zip/UI/Console/Main.cpp index e85364c3..8f2825ca 100644 --- a/CPP/7zip/UI/Console/Main.cpp +++ b/CPP/7zip/UI/Console/Main.cpp @@ -10,10 +10,6 @@ #include "../../../../C/CpuArch.h" -#if defined( _7ZIP_LARGE_PAGES) -#include "../../../../C/Alloc.h" -#endif - #include "../../../Common/MyInitGuid.h" #include "../../../Common/CommandLineParser.h" @@ -25,10 +21,6 @@ #include "../../../Windows/ErrorMsg.h" -#ifdef _WIN32 -#include "../../../Windows/MemoryLock.h" -#endif - #include "../../../Windows/TimeUtils.h" #include "../Common/ArchiveCommandLine.h" @@ -64,7 +56,7 @@ using namespace NCommandLineParser; HINSTANCE g_hInstance = 0; #endif -bool g_LargePagesMode = false; +extern bool g_LargePagesMode; extern CStdOutStream *g_StdStream; extern CStdOutStream *g_ErrStream; @@ -150,6 +142,7 @@ static const char * const kHelpString = " -spe : eliminate duplication of root folder for extract command\n" " -spf : use fully qualified file paths\n" " -ssc[-] : set sensitive case mode\n" + " -sse : stop archive creating, if it can't open some input file\n" " -ssw : compress shared files\n" " -stl : set archive timestamp from the most recently modified file\n" " -stm{HexMask} : set CPU thread affinity mask (hexadecimal number)\n" @@ -243,7 +236,8 @@ static void PrintWarningsPaths(const CErrorPathCodes &pc, CStdOutStream &so) { FOR_VECTOR(i, pc.Paths) { - so << pc.Paths[i] << " : "; + so.NormalizePrint_UString(pc.Paths[i]); + so << " : "; so << NError::MyFormatMessage(pc.Codes[i]) << endl; } so << "----------------" << endl; @@ -382,6 +376,8 @@ static void PrintMemUsage(const char *s, UInt64 val) *g_StdStream << " " << s << " Memory ="; PrintNum(SHIFT_SIZE_VALUE(val, 20), 7); *g_StdStream << " MB"; + if (g_LargePagesMode) + *g_StdStream << " (LP)"; } EXTERN_C_BEGIN @@ -523,6 +519,8 @@ int Main2( parser.Parse1(commandStrings, options); + g_StdOut.IsTerminalMode = options.IsStdOutTerminal; + g_StdErr.IsTerminalMode = options.IsStdErrTerminal; if (options.Number_for_Out != k_OutStream_stdout) g_StdStream = (options.Number_for_Out == k_OutStream_stderr ? &g_StdErr : NULL); @@ -540,24 +538,6 @@ int Main2( return 0; } - #if defined(_WIN32) && !defined(UNDER_CE) - NSecurity::EnablePrivilege_SymLink(); - #endif - - #ifdef _7ZIP_LARGE_PAGES - if (options.LargePages) - { - SetLargePageSize(); - // note: this process also can inherit that Privilege from parent process - g_LargePagesMode = - #if defined(_WIN32) && !defined(UNDER_CE) - NSecurity::EnablePrivilege_LockMemory(); - #else - true; - #endif - } - #endif - if (options.EnableHeaders) ShowCopyrightAndHelp(g_StdStream, false); diff --git a/CPP/7zip/UI/Console/MainAr.cpp b/CPP/7zip/UI/Console/MainAr.cpp index 25dabf2f..fdac6434 100644 --- a/CPP/7zip/UI/Console/MainAr.cpp +++ b/CPP/7zip/UI/Console/MainAr.cpp @@ -79,7 +79,7 @@ int MY_CDECL main PrintError(kUserBreakMessage); return (NExitCode::kUserBreak); } - catch(const CArcCmdLineException &e) + catch(const CMessagePathException &e) { PrintError(kException_CmdLine_Error_Message); if (g_ErrStream) diff --git a/CPP/7zip/UI/Console/PercentPrinter.cpp b/CPP/7zip/UI/Console/PercentPrinter.cpp index f94d8275..20249ed0 100644 --- a/CPP/7zip/UI/Console/PercentPrinter.cpp +++ b/CPP/7zip/UI/Console/PercentPrinter.cpp @@ -141,8 +141,9 @@ void CPercentPrinter::Print() { _s += ' '; - StdOut_Convert_UString_to_AString(FileName, _temp); - _temp.Replace('\n', ' '); + _tempU = FileName; + _so->Normalize_UString(_tempU); + StdOut_Convert_UString_to_AString(_tempU, _temp); if (_s.Len() + _temp.Len() > MaxLen) { unsigned len = FileName.Len(); @@ -153,8 +154,9 @@ void CPercentPrinter::Print() delta = 1; len -= delta; _tempU = FileName; - _tempU.Delete(len / 2, FileName.Len() - len); + _tempU.Delete(len / 2, _tempU.Len() - len); _tempU.Insert(len / 2, L" . "); + _so->Normalize_UString(_tempU); StdOut_Convert_UString_to_AString(_tempU, _temp); if (_s.Len() + _temp.Len() <= MaxLen) break; diff --git a/CPP/7zip/UI/Console/UpdateCallbackConsole.cpp b/CPP/7zip/UI/Console/UpdateCallbackConsole.cpp index 5e1c914a..46ffaba0 100644 --- a/CPP/7zip/UI/Console/UpdateCallbackConsole.cpp +++ b/CPP/7zip/UI/Console/UpdateCallbackConsole.cpp @@ -143,7 +143,9 @@ HRESULT CUpdateCallbackConsole::OpenResult( _so->Flush(); if (_se) { - *_se << kError << name << endl; + *_se << kError; + _se->NormalizePrint_wstr(name); + *_se << endl; HRESULT res = Print_OpenArchive_Error(*_se, codecs, arcLink); RINOK(res); _se->Flush(); @@ -185,7 +187,9 @@ void CCallbackConsoleBase::CommonError(const FString &path, DWORD systemError, b *_se << endl << (isWarning ? kWarning : kError) << NError::MyFormatMessage(systemError) - << endl << fs2us(path) << endl << endl; + << endl; + _se->NormalizePrint_UString(fs2us(path)); + *_se << endl << endl; _se->Flush(); } } @@ -281,8 +285,8 @@ HRESULT CUpdateCallbackConsole::StartArchive(const wchar_t *name, bool updating) if (_so) { *_so << (updating ? kUpdatingArchiveMessage : kCreatingArchiveMessage); - if (name != 0) - *_so << name; + if (name) + _so->NormalizePrint_wstr(name); else *_so << k_StdOut_ArcName; *_so << endl << endl; @@ -343,6 +347,7 @@ HRESULT CUpdateCallbackConsole::DeletingAfterArchiving(const FString &path, bool _tempA.Add_Space(); *_so << _tempA; _tempU = fs2us(path); + _so->Normalize_UString(_tempU); _so->PrintUString(_tempU, _tempA); *_so << endl; if (NeedFlush) @@ -475,7 +480,10 @@ HRESULT CCallbackConsoleBase::PrintProgress(const wchar_t *name, const char *com _tempU.Empty(); if (name) + { _tempU = name; + _so->Normalize_UString(_tempU); + } _so->PrintUString(_tempU, _tempA); *_so << endl; if (NeedFlush) @@ -568,7 +576,9 @@ HRESULT CUpdateCallbackConsole::ReportExtractResult(Int32 opRes, Int32 isEncrypt AString s; SetExtractErrorMessage(opRes, isEncrypted, s); - *_se << s << " : " << endl << name << endl << endl; + *_se << s << " : " << endl; + _se->NormalizePrint_wstr(name); + *_se << endl << endl; _se->Flush(); } return S_OK; diff --git a/CPP/7zip/UI/Console/resource.rc b/CPP/7zip/UI/Console/resource.rc index 8ffd50a9..8d721f50 100644 --- a/CPP/7zip/UI/Console/resource.rc +++ b/CPP/7zip/UI/Console/resource.rc @@ -1,3 +1,7 @@ #include "../../MyVersionInfo.rc" MY_VERSION_INFO_APP("7-Zip Console" , "7z") + +#ifndef UNDER_CE +1 24 MOVEABLE PURE "Console.manifest" +#endif diff --git a/CPP/7zip/UI/Explorer/Explorer.dsp b/CPP/7zip/UI/Explorer/Explorer.dsp index 4ca79a2b..7840943f 100644 --- a/CPP/7zip/UI/Explorer/Explorer.dsp +++ b/CPP/7zip/UI/Explorer/Explorer.dsp @@ -479,6 +479,14 @@ SOURCE=..\..\..\Windows\MemoryGlobal.h # End Source File # Begin Source File +SOURCE=..\..\..\Windows\MemoryLock.cpp +# End Source File +# Begin Source File + +SOURCE=..\..\..\Windows\MemoryLock.h +# End Source File +# Begin Source File + SOURCE=..\..\..\Windows\Menu.cpp # End Source File # Begin Source File diff --git a/CPP/7zip/UI/Explorer/makefile b/CPP/7zip/UI/Explorer/makefile index d82300d7..f916e4e8 100644 --- a/CPP/7zip/UI/Explorer/makefile +++ b/CPP/7zip/UI/Explorer/makefile @@ -34,6 +34,7 @@ WIN_OBJS = \ $O\FileFind.obj \ $O\FileIO.obj \ $O\FileName.obj \ + $O\MemoryLock.obj \ $O\Menu.obj \ $O\ProcessUtils.obj \ $O\Registry.obj \ diff --git a/CPP/7zip/UI/Far/ExtractEngine.cpp b/CPP/7zip/UI/Far/ExtractEngine.cpp index 18fced2c..20da0257 100644 --- a/CPP/7zip/UI/Far/ExtractEngine.cpp +++ b/CPP/7zip/UI/Far/ExtractEngine.cpp @@ -209,6 +209,8 @@ void SetExtractErrorMessage(Int32 opRes, Int32 encrypted, AString &s) s = "Is not archive"; else if (opRes == NArchive::NExtract::NOperationResult::kHeadersError) s = "kHeaders Error"; + else if (opRes == NArchive::NExtract::NOperationResult::kWrongPassword) + s = "Wrong Password"; else { s = "Error #"; diff --git a/CPP/7zip/UI/FileManager/EditDialog.cpp b/CPP/7zip/UI/FileManager/EditDialog.cpp new file mode 100644 index 00000000..965c7b17 --- /dev/null +++ b/CPP/7zip/UI/FileManager/EditDialog.cpp @@ -0,0 +1,57 @@ +// EditDialog.cpp + +#include "StdAfx.h" + +#include "EditDialog.h" + +#ifdef LANG +#include "LangUtils.h" +#endif + +bool CEditDialog::OnInit() +{ + #ifdef LANG + LangSetDlgItems(*this, NULL, 0); + #endif + _edit.Attach(GetItem(IDE_EDIT)); + + SetText(Title); + _edit.SetText(Text); + + NormalizeSize(); + return CModalDialog::OnInit(); +} + +// #define MY_CLOSE_BUTTON__ID IDCANCEL +#define MY_CLOSE_BUTTON__ID IDCLOSE + +bool CEditDialog::OnSize(WPARAM /* wParam */, int xSize, int ySize) +{ + int mx, my; + GetMargins(8, mx, my); + int bx1, by; + GetItemSizes(MY_CLOSE_BUTTON__ID, bx1, by); + + // int bx2; + // GetItemSizes(IDOK, bx2, by); + + int y = ySize - my - by; + int x = xSize - mx - bx1; + + /* + RECT rect; + GetClientRect(&rect); + rect.top = y - my; + InvalidateRect(&rect); + */ + InvalidateRect(NULL); + + MoveItem(MY_CLOSE_BUTTON__ID, x, y, bx1, by); + // MoveItem(IDOK, x - mx - bx2, y, bx2, by); + /* + if (wParam == SIZE_MAXSHOW || wParam == SIZE_MAXIMIZED || wParam == SIZE_MAXHIDE) + mx = 0; + */ + _edit.Move(mx, my, xSize - mx * 2, y - my * 2); + return false; +} diff --git a/CPP/7zip/UI/FileManager/EditDialog.h b/CPP/7zip/UI/FileManager/EditDialog.h new file mode 100644 index 00000000..b9cf08d6 --- /dev/null +++ b/CPP/7zip/UI/FileManager/EditDialog.h @@ -0,0 +1,25 @@ +// EditDialog.h + +#ifndef __EDIT_DIALOG_H +#define __EDIT_DIALOG_H + +#include "../../../Windows/Control/Dialog.h" +#include "../../../Windows/Control/Edit.h" + +#include "EditDialogRes.h" + +class CEditDialog: public NWindows::NControl::CModalDialog +{ + NWindows::NControl::CEdit _edit; + virtual bool OnInit(); + virtual bool OnSize(WPARAM wParam, int xSize, int ySize); +public: + UString Title; + UString Text; + + INT_PTR Create(HWND wndParent = 0) { return CModalDialog::Create(IDD_EDIT_DLG, wndParent); } + + CEditDialog() {} +}; + +#endif diff --git a/CPP/7zip/UI/FileManager/EditDialog.rc b/CPP/7zip/UI/FileManager/EditDialog.rc new file mode 100644 index 00000000..40d19966 --- /dev/null +++ b/CPP/7zip/UI/FileManager/EditDialog.rc @@ -0,0 +1,15 @@ +#include "EditDialogRes.h" +#include "../../GuiCommon.rc" + +#define xc 320 +#define yc 240 + +IDD_EDIT_DLG DIALOG 0, 0, xs, ys MY_MODAL_RESIZE_DIALOG_STYLE MY_FONT +CAPTION "Edit" +{ + // OK_CANCEL + MY_BUTTON__CLOSE + + EDITTEXT IDE_EDIT, m, m, xc, yc - bys - m, + ES_MULTILINE | ES_READONLY | WS_VSCROLL | WS_HSCROLL | ES_WANTRETURN +} diff --git a/CPP/7zip/UI/FileManager/EditDialogRes.h b/CPP/7zip/UI/FileManager/EditDialogRes.h new file mode 100644 index 00000000..1bbcf916 --- /dev/null +++ b/CPP/7zip/UI/FileManager/EditDialogRes.h @@ -0,0 +1,2 @@ +#define IDD_EDIT_DLG 94 +#define IDE_EDIT 100 diff --git a/CPP/7zip/UI/FileManager/FM.cpp b/CPP/7zip/UI/FileManager/FM.cpp index eb524254..43e45bb0 100644 --- a/CPP/7zip/UI/FileManager/FM.cpp +++ b/CPP/7zip/UI/FileManager/FM.cpp @@ -383,6 +383,7 @@ static void SetMemoryLock() NSecurity::AddLockMemoryPrivilege(); if (ReadLockMemoryEnable()) + if (NSecurity::Get_LargePages_RiskLevel() == 0) { // note: child processes can inherit that Privilege g_LargePagesMode = NSecurity::EnablePrivilege_LockMemory(); diff --git a/CPP/7zip/UI/FileManager/FM.dsp b/CPP/7zip/UI/FileManager/FM.dsp index e0a5b127..1e3b48a0 100644 --- a/CPP/7zip/UI/FileManager/FM.dsp +++ b/CPP/7zip/UI/FileManager/FM.dsp @@ -491,6 +491,14 @@ SOURCE=.\DialogSize.h # End Source File # Begin Source File +SOURCE=.\EditDialog.cpp +# End Source File +# Begin Source File + +SOURCE=.\EditDialog.h +# End Source File +# Begin Source File + SOURCE=.\LinkDialog.cpp # End Source File # Begin Source File diff --git a/CPP/7zip/UI/FileManager/FM.mak b/CPP/7zip/UI/FileManager/FM.mak index 0b99e846..4838dc17 100644 --- a/CPP/7zip/UI/FileManager/FM.mak +++ b/CPP/7zip/UI/FileManager/FM.mak @@ -46,6 +46,7 @@ FM_OBJS = \ $O\AboutDialog.obj \ $O\ComboDialog.obj \ $O\CopyDialog.obj \ + $O\EditDialog.obj \ $O\EditPage.obj \ $O\LangPage.obj \ $O\ListViewDialog.obj \ diff --git a/CPP/7zip/UI/FileManager/LangUtils.cpp b/CPP/7zip/UI/FileManager/LangUtils.cpp index bb448a7d..324dc362 100644 --- a/CPP/7zip/UI/FileManager/LangUtils.cpp +++ b/CPP/7zip/UI/FileManager/LangUtils.cpp @@ -58,6 +58,7 @@ static const CIDLangPair kLangPairs[] = { IDCANCEL, 402 }, { IDYES, 406 }, { IDNO, 407 }, + { IDCLOSE, 408 }, { IDHELP, 409 } }; diff --git a/CPP/7zip/UI/FileManager/LinkDialog.cpp b/CPP/7zip/UI/FileManager/LinkDialog.cpp index 9e69b522..a87bf622 100644 --- a/CPP/7zip/UI/FileManager/LinkDialog.cpp +++ b/CPP/7zip/UI/FileManager/LinkDialog.cpp @@ -55,7 +55,8 @@ static bool GetSymLink(CFSTR path, CReparseAttr &attr) if (!file.DeviceIoControlOut(my_FSCTL_GET_REPARSE_POINT, buf, kBufSize, &returnedSize)) return false; - if (!attr.Parse(buf, returnedSize)) + DWORD errorCode = 0; + if (!attr.Parse(buf, returnedSize, errorCode)) return false; CByteBuffer data2; @@ -291,7 +292,8 @@ void CLinkDialog::OnButton_Link() } CReparseAttr attr; - if (!attr.Parse(data, data.Size())) + DWORD errorCode = 0; + if (!attr.Parse(data, data.Size(), errorCode)) { ShowError(L"Internal conversion error"); return; diff --git a/CPP/7zip/UI/FileManager/ListViewDialog.cpp b/CPP/7zip/UI/FileManager/ListViewDialog.cpp index 2ede1a1e..79f566f2 100644 --- a/CPP/7zip/UI/FileManager/ListViewDialog.cpp +++ b/CPP/7zip/UI/FileManager/ListViewDialog.cpp @@ -2,6 +2,9 @@ #include "StdAfx.h" +#include "../../../Windows/Clipboard.h" + +#include "EditDialog.h" #include "ListViewDialog.h" #include "RegistryUtils.h" @@ -11,6 +14,23 @@ using namespace NWindows; +static const unsigned kOneStringMaxSize = 1024; + + +static void ListView_GetSelected(NControl::CListView &listView, CUIntVector &vector) +{ + vector.Clear(); + int index = -1; + for (;;) + { + index = listView.GetNextSelectedItem(index); + if (index < 0) + break; + vector.Add(index); + } +} + + bool CListViewDialog::OnInit() { #ifdef LANG @@ -18,28 +38,92 @@ bool CListViewDialog::OnInit() #endif _listView.Attach(GetItem(IDL_LISTVIEW)); + if (NumColumns > 1) + { + LONG_PTR style = _listView.GetStyle(); + style &= ~(LONG_PTR)LVS_NOCOLUMNHEADER; + _listView.SetStyle(style); + } + CFmSettings st; st.Load(); + + DWORD exStyle = 0; + if (st.SingleClick) - _listView.SetExtendedListViewStyle(LVS_EX_ONECLICKACTIVATE | LVS_EX_TRACKSELECT); + exStyle |= LVS_EX_ONECLICKACTIVATE | LVS_EX_TRACKSELECT; + + exStyle |= LVS_EX_FULLROWSELECT; + if (exStyle != 0) + _listView.SetExtendedListViewStyle(exStyle); + SetText(Title); + const int kWidth = 400; + LVCOLUMN columnInfo; columnInfo.mask = LVCF_FMT | LVCF_WIDTH | LVCF_SUBITEM; columnInfo.fmt = LVCFMT_LEFT; columnInfo.iSubItem = 0; - columnInfo.cx = 200; + columnInfo.cx = kWidth; + columnInfo.pszText = NULL; // (TCHAR *)(const TCHAR *)""; // "Property" - _listView.InsertColumn(0, &columnInfo); + if (NumColumns > 1) + { + columnInfo.cx = 100; + /* + // Windows always uses LVCFMT_LEFT for first column. + // if we need LVCFMT_RIGHT, we can create dummy column and then remove it + // columnInfo.mask |= LVCF_TEXT; + _listView.InsertColumn(0, &columnInfo); + + columnInfo.iSubItem = 1; + columnInfo.fmt = LVCFMT_RIGHT; + _listView.InsertColumn(1, &columnInfo); + _listView.DeleteColumn(0); + */ + } + // else + _listView.InsertColumn(0, &columnInfo); + + if (NumColumns > 1) + { + // columnInfo.fmt = LVCFMT_LEFT; + columnInfo.cx = kWidth - columnInfo.cx; + columnInfo.iSubItem = 1; + // columnInfo.pszText = NULL; // (TCHAR *)(const TCHAR *)""; // "Value" + _listView.InsertColumn(1, &columnInfo); + } + + + UString s; + FOR_VECTOR (i, Strings) + { _listView.InsertItem(i, Strings[i]); - if (Strings.Size() > 0) + if (NumColumns > 1 && i < Values.Size()) + { + s = Values[i]; + if (s.Len() > kOneStringMaxSize) + { + s.DeleteFrom(kOneStringMaxSize); + s += " ..."; + } + s.Replace(L"\r\n", L" "); + s.Replace(L"\n", L" "); + _listView.SetSubItem(i, 1, s); + } + } + + if (SelectFirst && Strings.Size() > 0) _listView.SetItemState_FocusedSelected(0); _listView.SetColumnWidthAuto(0); + if (NumColumns > 1) + _listView.SetColumnWidthAuto(1); StringsWereChanged = false; NormalizeSize(); @@ -74,8 +158,97 @@ bool CListViewDialog::OnSize(WPARAM /* wParam */, int xSize, int ySize) return false; } + extern bool g_LVN_ITEMACTIVATE_Support; +void CListViewDialog::CopyToClipboard() +{ + CUIntVector indexes; + ListView_GetSelected(_listView, indexes); + UString s; + + FOR_VECTOR (i, indexes) + { + unsigned index = indexes[i]; + s += Strings[index]; + if (NumColumns > 1 && index < Values.Size()) + { + const UString &v = Values[index]; + // if (!v.IsEmpty()) + { + s += ": "; + s += v; + } + } + // if (indexes.Size() > 1) + { + s += + #ifdef _WIN32 + "\r\n" + #else + "\n" + #endif + ; + } + } + + ClipboardSetText(*this, s); +} + + +void CListViewDialog::ShowItemInfo() +{ + CUIntVector indexes; + ListView_GetSelected(_listView, indexes); + if (indexes.Size() != 1) + return; + unsigned index = indexes[0]; + + CEditDialog dlg; + if (NumColumns == 1) + dlg.Text = Strings[index]; + else + { + dlg.Title = Strings[index]; + if (index < Values.Size()) + dlg.Text = Values[index]; + } + dlg.Create(*this); +} + + +void CListViewDialog::DeleteItems() +{ + for (;;) + { + int index = _listView.GetNextSelectedItem(-1); + if (index < 0) + break; + StringsWereChanged = true; + _listView.DeleteItem(index); + if ((unsigned)index < Strings.Size()) + Strings.Delete(index); + if ((unsigned)index < Values.Size()) + Values.Delete(index); + } + int focusedIndex = _listView.GetFocusedItem(); + if (focusedIndex >= 0) + _listView.SetItemState_FocusedSelected(focusedIndex); + _listView.SetColumnWidthAuto(0); +} + + +void CListViewDialog::OnEnter() +{ + if (IsKeyDown(VK_MENU) + || NumColumns > 1) + { + ShowItemInfo(); + return; + } + OnOK(); +} + bool CListViewDialog::OnNotify(UINT /* controlID */, LPNMHDR header) { if (header->hwndFrom != _listView) @@ -85,7 +258,7 @@ bool CListViewDialog::OnNotify(UINT /* controlID */, LPNMHDR header) case LVN_ITEMACTIVATE: if (g_LVN_ITEMACTIVATE_Support) { - OnOK(); + OnEnter(); return true; } break; @@ -93,7 +266,7 @@ bool CListViewDialog::OnNotify(UINT /* controlID */, LPNMHDR header) case NM_RETURN: // probabably it's unused if (!g_LVN_ITEMACTIVATE_Support) { - OnOK(); + OnEnter(); return true; } break; @@ -107,19 +280,7 @@ bool CListViewDialog::OnNotify(UINT /* controlID */, LPNMHDR header) { if (!DeleteIsAllowed) return false; - for (;;) - { - int index = _listView.GetNextSelectedItem(-1); - if (index < 0) - break; - StringsWereChanged = true; - _listView.DeleteItem(index); - Strings.Delete(index); - } - int focusedIndex = _listView.GetFocusedItem(); - if (focusedIndex >= 0) - _listView.SetItemState_FocusedSelected(focusedIndex); - _listView.SetColumnWidthAuto(0); + DeleteItems(); return true; } case 'A': @@ -129,6 +290,17 @@ bool CListViewDialog::OnNotify(UINT /* controlID */, LPNMHDR header) _listView.SelectAll(); return true; } + break; + } + case VK_INSERT: + case 'C': + { + if (IsKeyDown(VK_CONTROL)) + { + CopyToClipboard(); + return true; + } + break; } } } diff --git a/CPP/7zip/UI/FileManager/ListViewDialog.h b/CPP/7zip/UI/FileManager/ListViewDialog.h index f52be9e9..d18dae39 100644 --- a/CPP/7zip/UI/FileManager/ListViewDialog.h +++ b/CPP/7zip/UI/FileManager/ListViewDialog.h @@ -15,16 +15,32 @@ class CListViewDialog: public NWindows::NControl::CModalDialog virtual bool OnInit(); virtual bool OnSize(WPARAM wParam, int xSize, int ySize); virtual bool OnNotify(UINT controlID, LPNMHDR header); + void CopyToClipboard(); + void DeleteItems(); + void ShowItemInfo(); + void OnEnter(); public: UString Title; + + bool SelectFirst; bool DeleteIsAllowed; bool StringsWereChanged; + UStringVector Strings; + UStringVector Values; + int FocusedItemIndex; + unsigned NumColumns; INT_PTR Create(HWND wndParent = 0) { return CModalDialog::Create(IDD_LISTVIEW, wndParent); } - CListViewDialog(): DeleteIsAllowed(false) {} + CListViewDialog(): + SelectFirst(false), + DeleteIsAllowed(false), + StringsWereChanged(false), + FocusedItemIndex(-1), + NumColumns(1) + {} }; #endif diff --git a/CPP/7zip/UI/FileManager/ListViewDialog.rc b/CPP/7zip/UI/FileManager/ListViewDialog.rc index 85047962..0846c73a 100644 --- a/CPP/7zip/UI/FileManager/ListViewDialog.rc +++ b/CPP/7zip/UI/FileManager/ListViewDialog.rc @@ -1,8 +1,8 @@ #include "ListViewDialogRes.h" #include "../../GuiCommon.rc" -#define xc 320 -#define yc 240 +#define xc 440 +#define yc 320 IDD_LISTVIEW DIALOG 0, 0, xs, ys MY_MODAL_RESIZE_DIALOG_STYLE MY_FONT CAPTION "ListView" diff --git a/CPP/7zip/UI/FileManager/PanelCopy.cpp b/CPP/7zip/UI/FileManager/PanelCopy.cpp index 6f7555cb..bc69db64 100644 --- a/CPP/7zip/UI/FileManager/PanelCopy.cpp +++ b/CPP/7zip/UI/FileManager/PanelCopy.cpp @@ -16,7 +16,11 @@ using namespace NWindows; class CPanelCopyThread: public CProgressThreadVirt { + bool ResultsWereShown; + bool NeedShowRes; + HRESULT ProcessVirt(); + virtual void ProcessWasFinished_GuiVirt(); public: const CCopyToOptions *options; CMyComPtr FolderOperations; @@ -28,11 +32,31 @@ public: UString FirstFilePath; HRESULT Result; - - CPanelCopyThread(): Result(E_FAIL) {} -}; + void ShowFinalResults(HWND hwnd); + CPanelCopyThread(): + Result(E_FAIL), + ResultsWereShown(false), + NeedShowRes(false) + {} +}; + +void CPanelCopyThread::ShowFinalResults(HWND hwnd) +{ + if (NeedShowRes) + if (!ResultsWereShown) + { + ResultsWereShown = true; + ShowHashResults(Hash, fs2us(FirstFilePath), hwnd); + } +} + +void CPanelCopyThread::ProcessWasFinished_GuiVirt() +{ + ShowFinalResults(*this); +} + HRESULT CPanelCopyThread::ProcessVirt() { /* @@ -69,11 +93,15 @@ HRESULT CPanelCopyThread::ProcessVirt() BoolToInt(options->replaceAltStreamChars), options->folder, ExtractCallback); - if (Result == S_OK && !ExtractCallbackSpec->ThereAreMessageErrors && - (!options->hashMethods.IsEmpty() || options->testMode)) + if (Result == S_OK && !ExtractCallbackSpec->ThereAreMessageErrors) { - CProgressMessageBoxPair &pair = GetMessagePair(false); // GetMessagePair(ExtractCallbackSpec->Hash.NumErrors != 0); - AddHashBundleRes(pair.Message, Hash, FirstFilePath); + if (!options->hashMethods.IsEmpty()) + NeedShowRes = true; + else if (options->testMode) + { + CProgressMessageBoxPair &pair = GetMessagePair(false); // GetMessagePair(ExtractCallbackSpec->Hash.NumErrors != 0); + AddHashBundleRes(pair.Message, Hash, FirstFilePath); + } } return Result; @@ -92,7 +120,6 @@ static void ThrowException_if_Error(HRESULT res) #endif */ - HRESULT CPanel::CopyTo(CCopyToOptions &options, const CRecordVector &indices, UStringVector *messages, bool &usePassword, UString &password) @@ -102,7 +129,7 @@ HRESULT CPanel::CopyTo(CCopyToOptions &options, const CRecordVector &ind UString errorMessage = LangString(IDS_OPERATION_IS_NOT_SUPPORTED); if (options.showErrorMessages) MessageBox_Error(errorMessage); - else if (messages != 0) + else if (messages) messages->Add(errorMessage); return E_FAIL; } @@ -124,8 +151,8 @@ HRESULT CPanel::CopyTo(CCopyToOptions &options, const CRecordVector &ind extracter.ExtractCallback = extracter.ExtractCallbackSpec; extracter.options = &options; - extracter.ExtractCallbackSpec->ProgressDialog = &extracter.ProgressDialog; - extracter.ProgressDialog.CompressingMode = false; + extracter.ExtractCallbackSpec->ProgressDialog = &extracter; + extracter.CompressingMode = false; extracter.ExtractCallbackSpec->StreamMode = options.streamMode; @@ -185,9 +212,9 @@ HRESULT CPanel::CopyTo(CCopyToOptions &options, const CRecordVector &ind UString progressWindowTitle ("7-Zip"); // LangString(IDS_APP_TITLE); - extracter.ProgressDialog.MainWindow = GetParent(); - extracter.ProgressDialog.MainTitle = progressWindowTitle; - extracter.ProgressDialog.MainAddTitle = title + L' '; + extracter.MainWindow = GetParent(); + extracter.MainTitle = progressWindowTitle; + extracter.MainAddTitle = title + L' '; extracter.ExtractCallbackSpec->OverwriteMode = NExtract::NOverwriteMode::kAsk; extracter.ExtractCallbackSpec->Init(); @@ -199,8 +226,9 @@ HRESULT CPanel::CopyTo(CCopyToOptions &options, const CRecordVector &ind RINOK(extracter.Create(title, GetParent())); - if (messages != 0) - *messages = extracter.ProgressDialog.Sync.Messages; + + if (messages) + *messages = extracter.Sync.Messages; res = extracter.Result; if (res == S_OK && extracter.ExtractCallbackSpec->IsOK()) @@ -208,6 +236,9 @@ HRESULT CPanel::CopyTo(CCopyToOptions &options, const CRecordVector &ind usePassword = extracter.ExtractCallbackSpec->PasswordIsDefined; password = extracter.ExtractCallbackSpec->Password; } + + extracter.ShowFinalResults(_window); + } RefreshTitleAlways(); @@ -296,8 +327,8 @@ HRESULT CPanel::CopyFrom(bool moveMode, const UString &folderPrefix, const UStri NWindows::CThread thread; RINOK(thread.Create(CThreadUpdate::MyThreadFunction, &updater)); updater.ProgressDialog.Create(title, thread, GetParent()); - - if (messages != 0) + + if (messages) *messages = updater.ProgressDialog.Sync.Messages; res = updater.Result; @@ -308,7 +339,7 @@ HRESULT CPanel::CopyFrom(bool moveMode, const UString &folderPrefix, const UStri UString errorMessage = LangString(IDS_OPERATION_IS_NOT_SUPPORTED); if (showErrorMessages) MessageBox_Error(errorMessage); - else if (messages != 0) + else if (messages) messages->Add(errorMessage); return E_ABORT; } diff --git a/CPP/7zip/UI/FileManager/PanelCrc.cpp b/CPP/7zip/UI/FileManager/PanelCrc.cpp index 90443078..6353a722 100644 --- a/CPP/7zip/UI/FileManager/PanelCrc.cpp +++ b/CPP/7zip/UI/FileManager/PanelCrc.cpp @@ -139,18 +139,44 @@ DWORD CDirEnumerator::GetNextFile(NFind::CFileInfo &fi, bool &filled, FString &r class CThreadCrc: public CProgressThreadVirt { + bool ResultsWereShown; + bool WasFinished; + HRESULT ProcessVirt(); + virtual void ProcessWasFinished_GuiVirt(); public: CDirEnumerator Enumerator; CHashBundle Hash; + FString FirstFilePath; void SetStatus(const UString &s); void AddErrorMessage(DWORD systemError, const FChar *name); + void ShowFinalResults(HWND hwnd); + + CThreadCrc(): + ResultsWereShown(false), + WasFinished(false) + {} }; +void CThreadCrc::ShowFinalResults(HWND hwnd) +{ + if (WasFinished) + if (!ResultsWereShown) + { + ResultsWereShown = true; + ShowHashResults(Hash, fs2us(FirstFilePath), hwnd); + } +} + +void CThreadCrc::ProcessWasFinished_GuiVirt() +{ + ShowFinalResults(*this); +} + void CThreadCrc::AddErrorMessage(DWORD systemError, const FChar *name) { - ProgressDialog.Sync.AddError_Code_Name(systemError, fs2us(Enumerator.BasePrefix + name)); + Sync.AddError_Code_Name(systemError, fs2us(Enumerator.BasePrefix + name)); Hash.NumErrors++; } @@ -162,7 +188,7 @@ void CThreadCrc::SetStatus(const UString &s2) s.Add_Space_if_NotEmpty(); s += fs2us(Enumerator.BasePrefix); } - ProgressDialog.Sync.Set_Status(s); + Sync.Set_Status(s); } HRESULT CThreadCrc::ProcessVirt() @@ -173,7 +199,7 @@ HRESULT CThreadCrc::ProcessVirt() if (!buf.Allocate(kBufSize)) return E_OUTOFMEMORY; - CProgressSync &sync = ProgressDialog.Sync; + CProgressSync &sync = Sync; SetStatus(LangString(IDS_SCANNING)); @@ -233,7 +259,6 @@ HRESULT CThreadCrc::ProcessVirt() Enumerator.Init(); FString tempPath; - FString firstFilePath; bool isFirstFile = true; UInt64 errorsFilesSize = 0; @@ -264,7 +289,7 @@ HRESULT CThreadCrc::ProcessVirt() } if (isFirstFile) { - firstFilePath = path; + FirstFilePath = path; isFirstFile = false; } sync.Set_FilePath(fs2us(path)); @@ -303,12 +328,13 @@ HRESULT CThreadCrc::ProcessVirt() SetStatus(L""); CProgressMessageBoxPair &pair = GetMessagePair(Hash.NumErrors != 0); - AddHashBundleRes(pair.Message, Hash, fs2us(firstFilePath)); + WasFinished = true; LangString(IDS_CHECKSUM_INFORMATION, pair.Title); return S_OK; } + HRESULT CApp::CalculateCrc2(const UString &methodName) { unsigned srcPanelIndex = GetFocusedPanelIndex(); @@ -338,6 +364,7 @@ HRESULT CApp::CalculateCrc2(const UString &methodName) { CThreadCrc t; + { UStringVector methods; methods.Add(methodName); @@ -360,17 +387,20 @@ HRESULT CApp::CalculateCrc2(const UString &methodName) t.Enumerator.EnterToDirs = !GetFlatMode(); - t.ProgressDialog.ShowCompressionInfo = false; + t.ShowCompressionInfo = false; UString title = LangString(IDS_CHECKSUM_CALCULATING); - t.ProgressDialog.MainWindow = _window; - t.ProgressDialog.MainTitle = "7-Zip"; // LangString(IDS_APP_TITLE); - t.ProgressDialog.MainAddTitle = title; - t.ProgressDialog.MainAddTitle.Add_Space(); + t.MainWindow = _window; + t.MainTitle = "7-Zip"; // LangString(IDS_APP_TITLE); + t.MainAddTitle = title; + t.MainAddTitle.Add_Space(); RINOK(t.Create(title, _window)); + + t.ShowFinalResults(_window); } + RefreshTitleAlways(); return S_OK; } diff --git a/CPP/7zip/UI/FileManager/PanelDrag.cpp b/CPP/7zip/UI/FileManager/PanelDrag.cpp index 105a94b1..f28d3eb0 100644 --- a/CPP/7zip/UI/FileManager/PanelDrag.cpp +++ b/CPP/7zip/UI/FileManager/PanelDrag.cpp @@ -81,15 +81,17 @@ CDataObject::CDataObject() STDMETHODIMP CDataObject::SetData(LPFORMATETC etc, STGMEDIUM *medium, BOOL /* release */) { - if (etc->cfFormat == m_SetFolderFormat && etc->tymed == TYMED_HGLOBAL && - etc->dwAspect == DVASPECT_CONTENT && medium->tymed == TYMED_HGLOBAL) + if (etc->cfFormat == m_SetFolderFormat + && etc->tymed == TYMED_HGLOBAL + && etc->dwAspect == DVASPECT_CONTENT + && medium->tymed == TYMED_HGLOBAL) { Path.Empty(); - if (medium->hGlobal == 0) + if (!medium->hGlobal) return S_OK; size_t size = GlobalSize(medium->hGlobal) / sizeof(wchar_t); const wchar_t *src = (const wchar_t *)GlobalLock(medium->hGlobal); - if (src != 0) + if (src) { for (size_t i = 0; i < size; i++) { @@ -109,13 +111,13 @@ static HGLOBAL DuplicateGlobalMem(HGLOBAL srcGlobal) { SIZE_T size = GlobalSize(srcGlobal); const void *src = GlobalLock(srcGlobal); - if (src == 0) + if (!src) return 0; HGLOBAL destGlobal = GlobalAlloc(GHND | GMEM_SHARE, size); - if (destGlobal != 0) + if (destGlobal) { void *dest = GlobalLock(destGlobal); - if (dest == 0) + if (!dest) { GlobalFree(destGlobal); destGlobal = 0; @@ -136,7 +138,7 @@ STDMETHODIMP CDataObject::GetData(LPFORMATETC etc, LPSTGMEDIUM medium) medium->tymed = m_Etc.tymed; medium->pUnkForRelease = 0; medium->hGlobal = DuplicateGlobalMem(hGlobal); - if (medium->hGlobal == 0) + if (!medium->hGlobal) return E_OUTOFMEMORY; return S_OK; } @@ -261,7 +263,7 @@ static bool CopyNamesToHGlobal(NMemory::CGlobal &hgDrop, const UStringVector &na NMemory::CGlobalLock dropLock(hgDrop); DROPFILES* dropFiles = (DROPFILES*)dropLock.GetPointer(); - if (dropFiles == 0) + if (!dropFiles) return false; dropFiles->fNC = FALSE; dropFiles->pt.x = 0; @@ -291,7 +293,7 @@ static bool CopyNamesToHGlobal(NMemory::CGlobal &hgDrop, const UStringVector &na NMemory::CGlobalLock dropLock(hgDrop); DROPFILES* dropFiles = (DROPFILES*)dropLock.GetPointer(); - if (dropFiles == 0) + if (!dropFiles) return false; dropFiles->fNC = FALSE; dropFiles->pt.x = 0; @@ -336,7 +338,11 @@ void CPanel::OnDrag(LPNMLISTVIEW /* nmListView */) dirPrefix = us2fs(GetFsPath()); else { - tempDirectory.Create(kTempDirPrefix); + if (!tempDirectory.Create(kTempDirPrefix)) + { + MessageBox_Error(L"Can't create temp folder"); + return; + } dirPrefix = tempDirectory.GetPath(); // dirPrefix2 = dirPrefix; NFile::NName::NormalizeDirPathPrefix(dirPrefix); @@ -394,11 +400,67 @@ void CPanel::OnDrag(LPNMLISTVIEW /* nmListView */) dropSourceSpec->DataObjectSpec = dataObjectSpec; dropSourceSpec->DataObject = dataObjectSpec; - bool moveIsAllowed = isFSFolder; + + /* + CTime - file creation timestamp. + There are two operations in Windows with Drag and Drop: + COPY_OPERATION - icon with Plus sign - CTime will be set as current_time. + MOVE_OPERATION - icon without Plus sign - CTime will be preserved + Note: if we call DoDragDrop() with (effectsOK = DROPEFFECT_MOVE), then + it will use MOVE_OPERATION and CTime will be preserved. + But MoveFile() function doesn't preserve CTime, if different volumes are used. + Why it's so? + Does DoDragDrop() use some another function (not MoveFile())? + + if (effectsOK == DROPEFFECT_COPY) it works as COPY_OPERATION + + if (effectsOK == DROPEFFECT_MOVE) drag works as MOVE_OPERATION + + if (effectsOK == (DROPEFFECT_COPY | DROPEFFECT_MOVE)) + { + if we drag file to same volume, then Windows suggests: + CTRL - COPY_OPERATION + [default] - MOVE_OPERATION + + if we drag file to another volume, then Windows suggests + [default] - COPY_OPERATION + SHIFT - MOVE_OPERATION + } + + We want to use MOVE_OPERATION for extracting from archive (open in 7-Zip) to Explorer: + It has the following advantages: + 1) it uses fast MOVE_OPERATION instead of slow COPY_OPERATION and DELETE, if same volume. + 2) it preserved CTime + + Some another programs support only COPY_OPERATION. + So we can use (DROPEFFECT_COPY | DROPEFFECT_MOVE) + + Also another program can return from DoDragDrop() before + files using. But we delete temp folder after DoDragDrop(), + and another program can't open input files in that case. + + We create objects: + IDropSource *dropSource + IDataObject *dataObject + if DropTarget is 7-Zip window, then 7-Zip's + IDropTarget::DragOver() sets Path in IDataObject. + and + IDropSource::QueryContinueDrag() sets NeedPostCopy, if Path is not epmty. + So we can detect destination path after DoDragDrop(). + Now we don't know any good way to detect destination path for D&D to Explorer. + */ + + bool moveIsAllowed = isFSFolder; + /* DWORD effectsOK = DROPEFFECT_COPY; if (moveIsAllowed) effectsOK |= DROPEFFECT_MOVE; + */ + + // 18.04: was changed + DWORD effectsOK = DROPEFFECT_MOVE | DROPEFFECT_COPY; + DWORD effect; _panelCallback->DragBegin(); @@ -418,7 +480,8 @@ void CPanel::OnDrag(LPNMLISTVIEW /* nmListView */) NFile::NName::NormalizeDirPathPrefix(dataObjectSpec->Path); CCopyToOptions options; options.folder = dataObjectSpec->Path; - options.moveMode = (effect == DROPEFFECT_MOVE); + // if MOVE is not allowed, we just use COPY operation + options.moveMode = (effect == DROPEFFECT_MOVE && moveIsAllowed); res = CopyTo(options, indices, &dropSourceSpec->Messages); } /* @@ -475,7 +538,7 @@ static void MySetDropHighlighted(HWND hWnd, int index, bool enable) void CDropTarget::RemoveSelection() { - if (m_SelectionIndex >= 0 && m_Panel != 0) + if (m_SelectionIndex >= 0 && m_Panel) MySetDropHighlighted(m_Panel->_listView, m_SelectionIndex, false); m_SelectionIndex = -1; } @@ -555,7 +618,7 @@ void CDropTarget::PositionCursor(POINTL ptl) bool CDropTarget::IsFsFolderPath() const { - if (!m_IsAppTarget && m_Panel != 0) + if (!m_IsAppTarget && m_Panel) return (m_Panel->IsFSFolder() || (m_Panel->IsFSDrivesFolder() && m_SelectionIndex >= 0)); return false; } @@ -614,7 +677,7 @@ static void GetNamesFromDataObject(IDataObject *dataObject, UStringVector &names size_t blockSize = GlobalSize(medium.hGlobal); NMemory::CGlobalLock dropLock(medium.hGlobal); const DROPFILES* dropFiles = (DROPFILES*)dropLock.GetPointer(); - if (dropFiles == 0) + if (!dropFiles) return; if (blockSize < dropFiles->pFiles) return; @@ -629,7 +692,7 @@ static void GetNamesFromDataObject(IDataObject *dataObject, UStringVector &names bool CDropTarget::IsItSameDrive() const { - if (m_Panel == 0) + if (!m_Panel) return false; if (!IsFsFolderPath()) return false; @@ -662,6 +725,21 @@ bool CDropTarget::IsItSameDrive() const return true; } + +/* + There are 2 different actions, when we drag to 7-Zip: + 1) Drag from any external program except of Explorer to "7-Zip" FS folder. + We want to create new archive for that operation. + 2) all another operation work as usual file COPY/MOVE + - Drag from "7-Zip" FS to "7-Zip" FS. + COPY/MOVE are supported. + - Drag to open archive in 7-Zip. + We want to update archive. + We replace COPY to MOVE. + - Drag from "7-Zip" archive to "7-Zip" FS. + We replace COPY to MOVE. +*/ + DWORD CDropTarget::GetEffect(DWORD keyState, POINTL /* pt */, DWORD allowedEffect) { if (!m_DropIsAllowed || !m_PanelDropIsAllowed) @@ -671,10 +749,12 @@ DWORD CDropTarget::GetEffect(DWORD keyState, POINTL /* pt */, DWORD allowedEffec allowedEffect &= ~DROPEFFECT_MOVE; DWORD effect = 0; + if (keyState & MK_CONTROL) effect = allowedEffect & DROPEFFECT_COPY; else if (keyState & MK_SHIFT) effect = allowedEffect & DROPEFFECT_MOVE; + if (effect == 0) { if (allowedEffect & DROPEFFECT_COPY) @@ -716,10 +796,10 @@ bool CDropTarget::SetPath(bool enablePath) const path = GetTargetPath(); size_t size = path.Len() + 1; medium.hGlobal = GlobalAlloc(GHND | GMEM_SHARE, size * sizeof(wchar_t)); - if (medium.hGlobal == 0) + if (!medium.hGlobal) return false; wchar_t *dest = (wchar_t *)GlobalLock(medium.hGlobal); - if (dest == 0) + if (!dest) { GlobalUnlock(medium.hGlobal); return false; diff --git a/CPP/7zip/UI/FileManager/PanelFolderChange.cpp b/CPP/7zip/UI/FileManager/PanelFolderChange.cpp index ae6d420d..76f1f46f 100644 --- a/CPP/7zip/UI/FileManager/PanelFolderChange.cpp +++ b/CPP/7zip/UI/FileManager/PanelFolderChange.cpp @@ -625,6 +625,7 @@ void CPanel::FoldersHistory() { CListViewDialog listViewDialog; listViewDialog.DeleteIsAllowed = true; + listViewDialog.SelectFirst = true; LangString(IDS_FOLDERS_HISTORY, listViewDialog.Title); _appState->FolderHistory.GetList(listViewDialog.Strings); if (listViewDialog.Create(GetParent()) != IDOK) @@ -827,7 +828,8 @@ void CPanel::OpenFolder(int index) SetNewFolder(newFolder); LoadFullPath(); RefreshListCtrl(); - _listView.SetItemState_Selected(_listView.GetFocusedItem()); + // 17.02: fixed : now we don't select first item + // _listView.SetItemState_Selected(_listView.GetFocusedItem()); _listView.EnsureVisible(_listView.GetFocusedItem(), false); } diff --git a/CPP/7zip/UI/FileManager/PanelItemOpen.cpp b/CPP/7zip/UI/FileManager/PanelItemOpen.cpp index 8a83301c..1ff88881 100644 --- a/CPP/7zip/UI/FileManager/PanelItemOpen.cpp +++ b/CPP/7zip/UI/FileManager/PanelItemOpen.cpp @@ -674,7 +674,7 @@ static const char * const kStartExtensions = #endif " exe bat ps1 com" " chm" - " msi doc xls ppt pps wps wpt wks xlr wdb vsd pub" + " msi doc dot xls ppt pps wps wpt wks xlr wdb vsd pub" " docx docm dotx dotm xlsx xlsm xltx xltm xlsb xps" " xlam pptx pptm potx potm ppam ppsx ppsm xsn" @@ -1151,7 +1151,7 @@ HRESULT CPanel::OnOpenItemChanged(UInt32 index, const wchar_t *fullFilePath, CThreadCopyFrom t; t.UpdateCallbackSpec = new CUpdateCallback100Imp; t.UpdateCallback = t.UpdateCallbackSpec; - t.UpdateCallbackSpec->ProgressDialog = &t.ProgressDialog; + t.UpdateCallbackSpec->ProgressDialog = &t; t.ItemIndex = index; t.FullPath = fullFilePath; t.FolderOperations = _folderOperations; diff --git a/CPP/7zip/UI/FileManager/PanelListNotify.cpp b/CPP/7zip/UI/FileManager/PanelListNotify.cpp index d3d6139c..f6306de7 100644 --- a/CPP/7zip/UI/FileManager/PanelListNotify.cpp +++ b/CPP/7zip/UI/FileManager/PanelListNotify.cpp @@ -702,6 +702,8 @@ void CPanel::Refresh_StatusBar() wchar_t temp[32]; ConvertUInt32ToString(indices.Size(), temp); + wcscat(temp, L" / "); + ConvertUInt32ToString(_selectedStatusVector.Size(), temp + wcslen(temp)); // UString s1 = MyFormatNew(g_App.LangString_N_SELECTED_ITEMS, NumberToString(indices.Size())); // UString s1 = MyFormatNew(IDS_N_SELECTED_ITEMS, NumberToString(indices.Size())); diff --git a/CPP/7zip/UI/FileManager/PanelMenu.cpp b/CPP/7zip/UI/FileManager/PanelMenu.cpp index 7fd23486..d1161704 100644 --- a/CPP/7zip/UI/FileManager/PanelMenu.cpp +++ b/CPP/7zip/UI/FileManager/PanelMenu.cpp @@ -16,6 +16,7 @@ #include "App.h" #include "FormatUtils.h" #include "LangUtils.h" +#include "ListViewDialog.h" #include "MyLoadMenu.h" #include "PropertyName.h" @@ -50,17 +51,40 @@ void CPanel::InvokeSystemCommand(const char *command) contextMenu->InvokeCommand(&ci); } -static const char * const kSeparator = "----------------------------\n"; -static const char * const kSeparatorSmall = "----\n"; -static const char * const kPropValueSeparator = ": "; +static const char * const kSeparator = "------------------------"; +static const char * const kSeparatorSmall = "----------------"; extern UString ConvertSizeToString(UInt64 value) throw(); bool IsSizeProp(UINT propID) throw(); UString GetOpenArcErrorMessage(UInt32 errorFlags); + +static void AddListAscii(CListViewDialog &dialog, const char *s) +{ + dialog.Strings.Add((UString)s); + dialog.Values.AddNew(); +} + +static void AddSeparator(CListViewDialog &dialog) +{ + AddListAscii(dialog, kSeparator); +} + +static void AddSeparatorSmall(CListViewDialog &dialog) +{ + AddListAscii(dialog, kSeparatorSmall); +} + +static void AddPropertyPair(const UString &name, const UString &val, CListViewDialog &dialog) +{ + dialog.Strings.Add(name); + dialog.Values.Add(val); +} + + static void AddPropertyString(PROPID propID, const wchar_t *nameBSTR, - const NCOM::CPropVariant &prop, UString &s) + const NCOM::CPropVariant &prop, CListViewDialog &dialog) { if (prop.vt != VT_EMPTY) { @@ -87,23 +111,16 @@ static void AddPropertyString(PROPID propID, const wchar_t *nameBSTR, if (!val.IsEmpty()) { - s += GetNameOfProperty(propID, nameBSTR); - s += kPropValueSeparator; - /* - if (propID == kpidComment) - s.Add_LF(); - */ - s += val; - s.Add_LF(); + AddPropertyPair(GetNameOfProperty(propID, nameBSTR), val, dialog); } } } -static void AddPropertyString(PROPID propID, UInt64 val, UString &s) +static void AddPropertyString(PROPID propID, UInt64 val, CListViewDialog &dialog) { NCOM::CPropVariant prop = val; - AddPropertyString(propID, NULL, prop, s); + AddPropertyString(propID, NULL, prop, dialog); } @@ -137,7 +154,9 @@ void CPanel::Properties() } { - UString message; + CListViewDialog message; + // message.DeleteIsAllowed = false; + // message.SelectFirst = false; CRecordVector operatedIndices; GetOperatedItemIndices(operatedIndices); @@ -205,15 +224,12 @@ void CPanel::Properties() } } } - message += GetNameOfProperty(propID, name); - message += kPropValueSeparator; - message += s.Ptr(); - message.Add_LF(); + AddPropertyPair(GetNameOfProperty(propID, name), (UString)s.Ptr(), message); } } } - message += kSeparator; + AddSeparator(message); } else if (operatedIndices.Size() >= 1) { @@ -239,8 +255,7 @@ void CPanel::Properties() { wchar_t temp[32]; ConvertUInt32ToString(operatedIndices.Size(), temp); - message += MyFormatNew(g_App.LangString_N_SELECTED_ITEMS, temp); - message.Add_LF(); + AddPropertyPair(L"", MyFormatNew(g_App.LangString_N_SELECTED_ITEMS, temp), message); } if (numDirs != 0) @@ -250,7 +265,7 @@ void CPanel::Properties() AddPropertyString(kpidSize, unpackSize, message); AddPropertyString(kpidPackSize, packSize, message); - message += kSeparator; + AddSeparator(message); } @@ -309,7 +324,7 @@ void CPanel::Properties() { const int kNumSpecProps = ARRAY_SIZE(kSpecProps); - message += kSeparator; + AddSeparator(message); for (Int32 i = -(int)kNumSpecProps; i < (Int32)numProps; i++) { @@ -334,7 +349,7 @@ void CPanel::Properties() UInt32 numProps; if (getProps->GetArcNumProps2(level, &numProps) == S_OK) { - message += kSeparatorSmall; + AddSeparatorSmall(message); for (Int32 i = 0; i < (Int32)numProps; i++) { CMyComBSTR name; @@ -352,10 +367,15 @@ void CPanel::Properties() } } } - ::MessageBoxW(*(this), message, LangString(IDS_PROPERTIES), MB_OK); + + message.Title = LangString(IDS_PROPERTIES); + message.NumColumns = 2; + message.Create(GetParent()); } } + + void CPanel::EditCut() { // InvokeSystemCommand("cut"); @@ -403,82 +423,117 @@ void CPanel::EditPaste() // InvokeSystemCommand("paste"); } + + +struct CFolderPidls +{ + LPITEMIDLIST parent; + CRecordVector items; + + CFolderPidls(): parent(NULL) {} + ~CFolderPidls() + { + FOR_VECTOR (i, items) + CoTaskMemFree(items[i]); + CoTaskMemFree(parent); + } +}; + + HRESULT CPanel::CreateShellContextMenu( const CRecordVector &operatedIndices, CMyComPtr &systemContextMenu) { systemContextMenu.Release(); - UString folderPath = GetFsPath(); + const UString folderPath = GetFsPath(); CMyComPtr desktopFolder; RINOK(::SHGetDesktopFolder(&desktopFolder)); if (!desktopFolder) { - // ShowMessage("Failed to get Desktop folder."); + // ShowMessage("Failed to get Desktop folder"); return E_FAIL; } - // Separate the file from the folder. - - - // Get a pidl for the folder the file - // is located in. - LPITEMIDLIST parentPidl; + CFolderPidls pidls; DWORD eaten; + + // if (folderPath.IsEmpty()), then ParseDisplayName returns pidls of "My Computer" RINOK(desktopFolder->ParseDisplayName( - GetParent(), 0, (wchar_t *)(const wchar_t *)folderPath, - &eaten, &parentPidl, 0)); + GetParent(), NULL, (wchar_t *)(const wchar_t *)folderPath, + &eaten, &pidls.parent, NULL)); + + /* + STRRET pName; + res = desktopFolder->GetDisplayNameOf(pidls.parent, SHGDN_NORMAL, &pName); + WCHAR dir[MAX_PATH]; + if (!SHGetPathFromIDListW(pidls.parent, dir)) + dir[0] = 0; + */ + + if (!pidls.parent) + return E_FAIL; + + if (operatedIndices.IsEmpty()) + { + // how to get IContextMenu, if there are no selected files? + return E_FAIL; + + /* + xp64 : + 1) we can't use GetUIObjectOf() with (numItems == 0), it throws exception + 2) we can't use desktopFolder->GetUIObjectOf() with absolute pidls of folder + context menu items are different in that case: + "Open / Explorer" for folder + "Delete" for "My Computer" icon + "Preperties" for "System" + */ + /* + parentFolder = desktopFolder; + pidls.items.AddInReserved(pidls.parent); + pidls.parent = NULL; + */ + + // CreateViewObject() doesn't show all context menu items + /* + HRESULT res = parentFolder->CreateViewObject( + GetParent(), IID_IContextMenu, (void**)&systemContextMenu); + */ + } - // Get an IShellFolder for the folder - // the file is located in. CMyComPtr parentFolder; - RINOK(desktopFolder->BindToObject(parentPidl, - 0, IID_IShellFolder, (void**)&parentFolder)); + RINOK(desktopFolder->BindToObject(pidls.parent, + NULL, IID_IShellFolder, (void**)&parentFolder)); if (!parentFolder) { - // ShowMessage("Invalid file name."); + // ShowMessage("Invalid file name"); return E_FAIL; } - // Get a pidl for the file itself. - CRecordVector pidls; - pidls.ClearAndReserve(operatedIndices.Size()); + pidls.items.ClearAndReserve(operatedIndices.Size()); FOR_VECTOR (i, operatedIndices) { LPITEMIDLIST pidl; - UString fileName = GetItemRelPath2(operatedIndices[i]); + const UString fileName = GetItemRelPath2(operatedIndices[i]); RINOK(parentFolder->ParseDisplayName(GetParent(), 0, - (wchar_t *)(const wchar_t *)fileName, &eaten, &pidl, 0)); - pidls.AddInReserved(pidl); + (wchar_t *)(const wchar_t *)fileName, &eaten, &pidl, 0)); + pidls.items.AddInReserved(pidl); } + + // Get IContextMenu for items - ITEMIDLIST temp; - if (pidls.Size() == 0) + RINOK(parentFolder->GetUIObjectOf(GetParent(), pidls.items.Size(), + (LPCITEMIDLIST *)&pidls.items.Front(), IID_IContextMenu, 0, (void**)&systemContextMenu)); + + if (!systemContextMenu) { - temp.mkid.cb = 0; - /* - LPITEMIDLIST pidl; - HRESULT result = parentFolder->ParseDisplayName(GetParent(), 0, - L"." WSTRING_PATH_SEPARATOR, &eaten, &pidl, 0); - if (result != NOERROR) - return; - */ - pidls.Add(&temp); - } - - // Get the IContextMenu for the file. - CMyComPtr cm; - RINOK( parentFolder->GetUIObjectOf(GetParent(), pidls.Size(), - (LPCITEMIDLIST *)&pidls.Front(), IID_IContextMenu, 0, (void**)&cm)); - if (!cm) - { - // ShowMessage("Unable to get context menu interface."); + // ShowMessage("Unable to get context menu interface"); return E_FAIL; } - systemContextMenu = cm; return S_OK; } + void CPanel::CreateSystemMenu(HMENU menuSpec, const CRecordVector &operatedIndices, CMyComPtr &systemContextMenu) diff --git a/CPP/7zip/UI/FileManager/PanelOperations.cpp b/CPP/7zip/UI/FileManager/PanelOperations.cpp index 1db4b21c..e61c57dd 100644 --- a/CPP/7zip/UI/FileManager/PanelOperations.cpp +++ b/CPP/7zip/UI/FileManager/PanelOperations.cpp @@ -79,10 +79,10 @@ HRESULT CThreadFolderOperations::DoOperation(CPanel &panel, const UString &progr { UpdateCallbackSpec = new CUpdateCallback100Imp; UpdateCallback = UpdateCallbackSpec; - UpdateCallbackSpec->ProgressDialog = &ProgressDialog; + UpdateCallbackSpec->ProgressDialog = this; - ProgressDialog.WaitMode = true; - ProgressDialog.Sync.FinalMessage.ErrorMessage.Title = titleError; + WaitMode = true; + Sync.FinalMessage.ErrorMessage.Title = titleError; Result = S_OK; UpdateCallbackSpec->Init(); @@ -95,11 +95,11 @@ HRESULT CThreadFolderOperations::DoOperation(CPanel &panel, const UString &progr } - ProgressDialog.MainWindow = panel._mainWindow; // panel.GetParent() - ProgressDialog.MainTitle = "7-Zip"; // LangString(IDS_APP_TITLE); - ProgressDialog.MainAddTitle = progressTitle + L' '; + MainWindow = panel._mainWindow; // panel.GetParent() + MainTitle = "7-Zip"; // LangString(IDS_APP_TITLE); + MainAddTitle = progressTitle + L' '; - RINOK(Create(progressTitle, ProgressDialog.MainWindow)); + RINOK(Create(progressTitle, MainWindow)); return Result; } diff --git a/CPP/7zip/UI/FileManager/PanelSelect.cpp b/CPP/7zip/UI/FileManager/PanelSelect.cpp index 4e179190..022eeb67 100644 --- a/CPP/7zip/UI/FileManager/PanelSelect.cpp +++ b/CPP/7zip/UI/FileManager/PanelSelect.cpp @@ -218,6 +218,8 @@ void CPanel::InvertSelection() FOR_VECTOR (i, _selectedStatusVector) if (_selectedStatusVector[i]) numSelected++; + // 17.02: fixed : now we invert item even, if single item is selected + /* if (numSelected == 1) { int focused = _listView.GetFocusedItem(); @@ -229,6 +231,7 @@ void CPanel::InvertSelection() _selectedStatusVector[realIndex] = false; } } + */ } FOR_VECTOR (i, _selectedStatusVector) _selectedStatusVector[i] = !_selectedStatusVector[i]; diff --git a/CPP/7zip/UI/FileManager/PanelSplitFile.cpp b/CPP/7zip/UI/FileManager/PanelSplitFile.cpp index d3d4b72e..f5f9aa2d 100644 --- a/CPP/7zip/UI/FileManager/PanelSplitFile.cpp +++ b/CPP/7zip/UI/FileManager/PanelSplitFile.cpp @@ -77,8 +77,6 @@ UString CVolSeqName::GetNextName() return UnchangedPart + ChangedPart; } -static const UInt32 kBufSize = (1 << 20); - class CThreadSplit: public CProgressThreadVirt { HRESULT ProcessVirt(); @@ -89,30 +87,84 @@ public: CRecordVector VolumeSizes; }; + +class CPreAllocOutFile +{ + UInt64 _preAllocSize; +public: + NIO::COutFile File; + UInt64 Written; + + CPreAllocOutFile(): _preAllocSize(0), Written(0) {} + + ~CPreAllocOutFile() + { + SetCorrectFileLength(); + } + + void PreAlloc(UInt64 preAllocSize) + { + _preAllocSize = 0; + if (File.SetLength(preAllocSize)) + _preAllocSize = preAllocSize; + File.SeekToBegin(); + } + + bool Write(const void *data, UInt32 size, UInt32 &processedSize) throw() + { + bool res = File.Write(data, size, processedSize); + Written += processedSize; + return res; + } + + void Close() + { + SetCorrectFileLength(); + Written = 0; + _preAllocSize = 0; + File.Close(); + } + + void SetCorrectFileLength() + { + if (Written < _preAllocSize) + { + File.SetLength(Written); + _preAllocSize = 0; + } + } +}; + + +static const UInt32 kBufSize = (1 << 20); + HRESULT CThreadSplit::ProcessVirt() { NIO::CInFile inFile; if (!inFile.Open(FilePath)) return GetLastError(); - NIO::COutFile outFile; - CMyBuffer bufferObject; - if (!bufferObject.Allocate(kBufSize)) + + CPreAllocOutFile outFile; + + CMyBuffer buffer; + if (!buffer.Allocate(kBufSize)) return E_OUTOFMEMORY; - Byte *buffer = (Byte *)(void *)bufferObject; - UInt64 curVolSize = 0; + CVolSeqName seqName; seqName.SetNumDigits(NumVolumes); + UInt64 length; if (!inFile.GetLength(length)) return GetLastError(); - CProgressSync &sync = ProgressDialog.Sync; + CProgressSync &sync = Sync; sync.Set_NumBytesTotal(length); - UInt64 pos = 0; + UInt64 pos = 0; + UInt64 prev = 0; UInt64 numFiles = 0; unsigned volIndex = 0; - + for (;;) { UInt64 volSize; @@ -121,46 +173,65 @@ HRESULT CThreadSplit::ProcessVirt() else volSize = VolumeSizes.Back(); - UInt32 needSize = (UInt32)(MyMin((UInt64)kBufSize, volSize - curVolSize)); + UInt32 needSize = kBufSize; + { + const UInt64 rem = volSize - outFile.Written; + if (needSize > rem) + needSize = (UInt32)rem; + } UInt32 processedSize; if (!inFile.Read(buffer, needSize, processedSize)) return GetLastError(); if (processedSize == 0) - break; + return S_OK; needSize = processedSize; - if (curVolSize == 0) + + if (outFile.Written == 0) { FString name = VolBasePath; name += '.'; name += us2fs(seqName.GetNextName()); sync.Set_FilePath(fs2us(name)); - sync.Set_NumFilesCur(numFiles++); - if (!outFile.Create(name, false)) + if (!outFile.File.Create(name, false)) { HRESULT res = GetLastError(); AddErrorPath(name); return res; } + UInt64 expectSize = volSize; + if (pos < length) + { + const UInt64 rem = length - pos; + if (expectSize > rem) + expectSize = rem; + } + outFile.PreAlloc(expectSize); } + if (!outFile.Write(buffer, needSize, processedSize)) return GetLastError(); if (needSize != processedSize) throw g_Message_FileWriteError; - curVolSize += processedSize; - if (curVolSize == volSize) + + pos += processedSize; + + if (outFile.Written == volSize) { outFile.Close(); + sync.Set_NumFilesCur(++numFiles); if (volIndex < VolumeSizes.Size()) volIndex++; - curVolSize = 0; } - pos += processedSize; - RINOK(sync.Set_NumBytesCur(pos)); + + if (pos - prev >= ((UInt32)1 << 22) || outFile.Written == 0) + { + RINOK(sync.Set_NumBytesCur(pos)); + prev = pos; + } } - sync.Set_NumFilesCur(numFiles); - return S_OK; } + void CApp::Split() { int srcPanelIndex = GetFocusedPanelIndex(); @@ -235,7 +306,7 @@ void CApp::Split() CThreadSplit spliter; spliter.NumVolumes = numVolumes; - CProgressDialog &progressDialog = spliter.ProgressDialog; + CProgressDialog &progressDialog = spliter; UString progressWindowTitle ("7-Zip"); // LangString(IDS_APP_TITLE, 0x03000000); UString title = LangString(IDS_SPLITTING); @@ -291,7 +362,7 @@ HRESULT CThreadCombine::ProcessVirt() return res; } - CProgressSync &sync = ProgressDialog.Sync; + CProgressSync &sync = Sync; sync.Set_NumBytesTotal(TotalSize); CMyBuffer bufferObject; @@ -463,7 +534,7 @@ void CApp::Combine() return; } - CProgressDialog &progressDialog = combiner.ProgressDialog; + CProgressDialog &progressDialog = combiner; progressDialog.ShowCompressionInfo = false; UString progressWindowTitle ("7-Zip"); // LangString(IDS_APP_TITLE, 0x03000000); diff --git a/CPP/7zip/UI/FileManager/ProgressDialog2.cpp b/CPP/7zip/UI/FileManager/ProgressDialog2.cpp index 92ee26a3..bdb2be3f 100644 --- a/CPP/7zip/UI/FileManager/ProgressDialog2.cpp +++ b/CPP/7zip/UI/FileManager/ProgressDialog2.cpp @@ -236,7 +236,10 @@ void CProgressSync::AddError_Code_Name(DWORD systemError, const wchar_t *name) AddError_Message_Name(s, name); } -CProgressDialog::CProgressDialog(): _timer(0), CompressingMode(true), MainWindow(0) +CProgressDialog::CProgressDialog(): + _timer(0), + CompressingMode(true), + MainWindow(0) { _isDir = false; @@ -974,7 +977,9 @@ bool CProgressDialog::OnExternalCloseMessage() ::SendMessage(GetItem(IDCANCEL), BM_SETSTYLE, BS_DEFPUSHBUTTON, MAKELPARAM(TRUE, 0)); HideItem(IDB_PROGRESS_BACKGROUND); HideItem(IDB_PAUSE); - + + ProcessWasFinished_GuiVirt(); + bool thereAreMessages; CProgressFinalMessage fm; { @@ -982,6 +987,7 @@ bool CProgressDialog::OnExternalCloseMessage() thereAreMessages = !Sync.Messages.IsEmpty(); fm = Sync.FinalMessage; } + if (!fm.ErrorMessage.Message.IsEmpty()) { MessagesDisplayed = true; @@ -992,6 +998,7 @@ bool CProgressDialog::OnExternalCloseMessage() else if (!thereAreMessages) { MessagesDisplayed = true; + if (!fm.OkMessage.Message.IsEmpty()) { if (fm.OkMessage.Title.IsEmpty()) @@ -1245,11 +1252,24 @@ void CProgressDialog::ProcessWasFinished() } +static THREAD_FUNC_DECL MyThreadFunction(void *param) +{ + CProgressThreadVirt *p = (CProgressThreadVirt *)param; + try + { + p->Process(); + p->ThreadFinishedOK = true; + } + catch (...) { p->Result = E_FAIL; } + return 0; +} + + HRESULT CProgressThreadVirt::Create(const UString &title, HWND parentWindow) { NWindows::CThread thread; RINOK(thread.Create(MyThreadFunction, this)); - ProgressDialog.Create(title, thread, parentWindow); + CProgressDialog::Create(title, thread, parentWindow); return S_OK; } @@ -1265,7 +1285,7 @@ static void AddMessageToString(UString &dest, const UString &src) void CProgressThreadVirt::Process() { - CProgressCloser closer(ProgressDialog); + CProgressCloser closer(*this); UString m; try { Result = ProcessVirt(); } catch(const wchar_t *s) { m = s; } @@ -1293,7 +1313,7 @@ void CProgressThreadVirt::Process() } } - CProgressSync &sync = ProgressDialog.Sync; + CProgressSync &sync = Sync; NSynchronization::CCriticalSectionLock lock(sync._cs); if (m.IsEmpty()) { diff --git a/CPP/7zip/UI/FileManager/ProgressDialog2.h b/CPP/7zip/UI/FileManager/ProgressDialog2.h index 0e2f2ce0..5e916e6f 100644 --- a/CPP/7zip/UI/FileManager/ProgressDialog2.h +++ b/CPP/7zip/UI/FileManager/ProgressDialog2.h @@ -261,7 +261,18 @@ public: INT_PTR Create(const UString &title, NWindows::CThread &thread, HWND wndParent = 0); + + /* how it works: + 1) the working thread calls ProcessWasFinished() + that sends kCloseMessage message to CProgressDialog (GUI) thread + 2) CProgressDialog (GUI) thread receives kCloseMessage message and + calls ProcessWasFinished_GuiVirt(); + So we can implement ProcessWasFinished_GuiVirt() and show special + results window in GUI thread with CProgressDialog as parent window + */ + void ProcessWasFinished(); + virtual void ProcessWasFinished_GuiVirt() {} }; @@ -273,7 +284,8 @@ public: ~CProgressCloser() { _p->ProcessWasFinished(); } }; -class CProgressThreadVirt + +class CProgressThreadVirt: public CProgressDialog { protected: FStringVector ErrorPaths; @@ -281,33 +293,59 @@ protected: // error if any of HRESULT, ErrorMessage, ErrorPath virtual HRESULT ProcessVirt() = 0; - void Process(); public: HRESULT Result; bool ThreadFinishedOK; // if there is no fatal exception - CProgressDialog ProgressDialog; - - static THREAD_FUNC_DECL MyThreadFunction(void *param) - { - CProgressThreadVirt *p = (CProgressThreadVirt *)param; - try - { - p->Process(); - p->ThreadFinishedOK = true; - } - catch (...) { p->Result = E_FAIL; } - return 0; - } + void Process(); void AddErrorPath(const FString &path) { ErrorPaths.Add(path); } HRESULT Create(const UString &title, HWND parentWindow = 0); CProgressThreadVirt(): Result(E_FAIL), ThreadFinishedOK(false) {} CProgressMessageBoxPair &GetMessagePair(bool isError) { return isError ? FinalMessage.ErrorMessage : FinalMessage.OkMessage; } - }; UString HResultToMessage(HRESULT errorCode); +/* +how it works: + +client code inherits CProgressThreadVirt and calls +CProgressThreadVirt::Create() +{ + it creates new thread that calls CProgressThreadVirt::Process(); + it creates modal progress dialog window with ProgressDialog.Create() +} + +CProgressThreadVirt::Process() +{ + { + ProcessVirt(); // virtual function that must implement real work + } + if (exceptions) or FinalMessage.ErrorMessage.Message + { + set message to ProgressDialog.Sync.FinalMessage.ErrorMessage.Message + } + else if (FinalMessage.OkMessage.Message) + { + set message to ProgressDialog.Sync.FinalMessage.OkMessage + } + + PostMsg(kCloseMessage); +} + + +CProgressDialog::OnExternalCloseMessage() +{ + if (ProgressDialog.Sync.FinalMessage) + { + WorkWasFinishedVirt(); + Show (ProgressDialog.Sync.FinalMessage) + MessagesDisplayed = true; + } +} + +*/ + #endif diff --git a/CPP/7zip/UI/FileManager/resource.rc b/CPP/7zip/UI/FileManager/resource.rc index 2efb26d5..5dbe55ab 100644 --- a/CPP/7zip/UI/FileManager/resource.rc +++ b/CPP/7zip/UI/FileManager/resource.rc @@ -247,6 +247,7 @@ END #include "BrowseDialog.rc" #include "ComboDialog.rc" #include "CopyDialog.rc" +#include "EditDialog.rc" #include "EditPage.rc" #include "FoldersPage.rc" #include "LangPage.rc" diff --git a/CPP/7zip/UI/GUI/CompressDialog.cpp b/CPP/7zip/UI/GUI/CompressDialog.cpp index e189e9b5..c3c8c72e 100644 --- a/CPP/7zip/UI/GUI/CompressDialog.cpp +++ b/CPP/7zip/UI/GUI/CompressDialog.cpp @@ -964,6 +964,7 @@ bool CCompressDialog::OnCommand(int code, int itemID, LPARAM lParam) { bool isSFX = IsSFX(); SaveOptionsInMem(); + m_Solid.ResetContent(); SetLevel(); SetSolidBlockSize(); SetNumThreads(); diff --git a/CPP/7zip/UI/GUI/ExtractGUI.cpp b/CPP/7zip/UI/GUI/ExtractGUI.cpp index c51d50e9..37aa45b3 100644 --- a/CPP/7zip/UI/GUI/ExtractGUI.cpp +++ b/CPP/7zip/UI/GUI/ExtractGUI.cpp @@ -52,22 +52,9 @@ static void AddValuePair(UString &s, UINT resourceID, UInt64 value, bool addColo static void AddSizePair(UString &s, UINT resourceID, UInt64 value) { - { - wchar_t sz[32]; - AddLangString(s, resourceID); - s += ": "; - ConvertUInt64ToString(value, sz); - s += MyFormatNew(IDS_FILE_SIZE, sz); - } - // s += sz; - if (value >= (1 << 20)) - { - char sz[32]; - ConvertUInt64ToString(value >> 20, sz); - s += " ("; - s += sz; - s += " MB)"; - } + AddLangString(s, resourceID); + s += ": "; + AddSizeValue(s, value); s.Add_LF(); } @@ -86,16 +73,31 @@ public: UStringVector *ArchivePathsFull; const NWildcard::CCensorNode *WildcardCensor; const CExtractOptions *Options; + #ifndef _SFX CHashBundle *HashBundle; + virtual void ProcessWasFinished_GuiVirt(); #endif + CMyComPtr ExtractCallback; UString Title; + + CPropNameValPairs Pairs; }; + +#ifndef _SFX +void CThreadExtracting::ProcessWasFinished_GuiVirt() +{ + if (HashBundle && !Pairs.IsEmpty()) + ShowHashResults(Pairs, *this); +} +#endif + HRESULT CThreadExtracting::ProcessVirt() { CDecompressStat Stat; + #ifndef _SFX if (HashBundle) HashBundle->Init(); @@ -109,16 +111,23 @@ HRESULT CThreadExtracting::ProcessVirt() HashBundle, #endif FinalMessage.ErrorMessage.Message, Stat); + #ifndef _SFX - if (res == S_OK && Options->TestMode && ExtractCallbackSpec->IsOK()) + if (res == S_OK && ExtractCallbackSpec->IsOK()) { - UString s; - - AddValuePair(s, IDS_ARCHIVES_COLON, Stat.NumArchives, false); - AddSizePair(s, IDS_PROP_PACKED_SIZE, Stat.PackSize); - - if (!HashBundle) + if (HashBundle) { + AddValuePair(Pairs, IDS_ARCHIVES_COLON, Stat.NumArchives); + AddSizeValuePair(Pairs, IDS_PROP_PACKED_SIZE, Stat.PackSize); + AddHashBundleRes(Pairs, *HashBundle, UString()); + } + else if (Options->TestMode) + { + UString s; + + AddValuePair(s, IDS_ARCHIVES_COLON, Stat.NumArchives, false); + AddSizePair(s, IDS_PROP_PACKED_SIZE, Stat.PackSize); + if (Stat.NumFolders != 0) AddValuePair(s, IDS_PROP_FOLDERS, Stat.NumFolders); AddValuePair(s, IDS_PROP_FILES, Stat.NumFiles); @@ -129,24 +138,19 @@ HRESULT CThreadExtracting::ProcessVirt() AddValuePair(s, IDS_PROP_NUM_ALT_STREAMS, Stat.NumAltStreams); AddSizePair(s, IDS_PROP_ALT_STREAMS_SIZE, Stat.AltStreams_UnpackSize); } - } - - if (HashBundle) - { s.Add_LF(); - AddHashBundleRes(s, *HashBundle, UString()); + AddLangString(s, IDS_MESSAGE_NO_ERRORS); + FinalMessage.OkMessage.Title = Title; + FinalMessage.OkMessage.Message = s; } - - s.Add_LF(); - AddLangString(s, IDS_MESSAGE_NO_ERRORS); - - FinalMessage.OkMessage.Title = Title; - FinalMessage.OkMessage.Message = s; } #endif + return res; } + + HRESULT ExtractGUI( CCodecs *codecs, const CObjectVector &formatIndices, @@ -252,11 +256,11 @@ HRESULT ExtractGUI( extracter.Title = title; extracter.ExtractCallbackSpec = extractCallback; - extracter.ExtractCallbackSpec->ProgressDialog = &extracter.ProgressDialog; + extracter.ExtractCallbackSpec->ProgressDialog = &extracter; extracter.ExtractCallback = extractCallback; extracter.ExtractCallbackSpec->Init(); - extracter.ProgressDialog.CompressingMode = false; + extracter.CompressingMode = false; extracter.ArchivePaths = &archivePaths; extracter.ArchivePathsFull = &archivePathsFull; @@ -266,10 +270,9 @@ HRESULT ExtractGUI( extracter.HashBundle = hb; #endif - extracter.ProgressDialog.IconID = IDI_ICON; + extracter.IconID = IDI_ICON; RINOK(extracter.Create(title, hwndParent)); - messageWasDisplayed = extracter.ThreadFinishedOK & - extracter.ProgressDialog.MessagesDisplayed; + messageWasDisplayed = extracter.ThreadFinishedOK && extracter.MessagesDisplayed; return extracter.Result; } diff --git a/CPP/7zip/UI/GUI/GUI.cpp b/CPP/7zip/UI/GUI/GUI.cpp index c5e43e79..f613f97d 100644 --- a/CPP/7zip/UI/GUI/GUI.cpp +++ b/CPP/7zip/UI/GUI/GUI.cpp @@ -6,8 +6,6 @@ #include -#include "../../../../C/Alloc.h" - #include "../../../Common/MyInitGuid.h" #include "../../../Common/CommandLineParser.h" @@ -16,9 +14,6 @@ #include "../../../Windows/FileDir.h" #include "../../../Windows/NtCheck.h" -#ifdef _WIN32 -#include "../../../Windows/MemoryLock.h" -#endif #include "../Common/ArchiveCommandLine.h" #include "../Common/ExitCode.h" @@ -37,8 +32,6 @@ using namespace NWindows; HINSTANCE g_hInstance; -bool g_LargePagesMode = false; - #ifndef UNDER_CE DWORD g_ComCtl32Version; @@ -126,24 +119,6 @@ static int Main2() parser.Parse1(commandStrings, options); parser.Parse2(options); - #if defined(_WIN32) && !defined(UNDER_CE) - NSecurity::EnablePrivilege_SymLink(); - #endif - - #ifdef _7ZIP_LARGE_PAGES - if (options.LargePages) - { - SetLargePageSize(); - // note: this process also can inherit that Privilege from parent process - g_LargePagesMode = - #if defined(_WIN32) && !defined(UNDER_CE) - NSecurity::EnablePrivilege_LockMemory(); - #else - true; - #endif - } - #endif - CREATE_CODECS_OBJECT codecs->CaseSensitiveChange = options.CaseSensitiveChange; @@ -403,7 +378,7 @@ int APIENTRY WinMain(HINSTANCE hInstance, HINSTANCE /* hPrevInstance */, { return ShowMemErrorMessage(); } - catch(const CArcCmdLineException &e) + catch(const CMessagePathException &e) { ErrorMessage(e); return NExitCode::kUserError; diff --git a/CPP/7zip/UI/GUI/GUI.dsp b/CPP/7zip/UI/GUI/GUI.dsp index 223b441f..e9aaa889 100644 --- a/CPP/7zip/UI/GUI/GUI.dsp +++ b/CPP/7zip/UI/GUI/GUI.dsp @@ -425,6 +425,14 @@ SOURCE=.\CompressDialog.h # End Source File # Begin Source File +SOURCE=..\FileManager\EditDialog.cpp +# End Source File +# Begin Source File + +SOURCE=..\FileManager\EditDialog.h +# End Source File +# Begin Source File + SOURCE=.\ExtractDialog.cpp # End Source File # Begin Source File @@ -433,6 +441,14 @@ SOURCE=.\ExtractDialog.h # End Source File # Begin Source File +SOURCE=..\FileManager\ListViewDialog.cpp +# End Source File +# Begin Source File + +SOURCE=..\FileManager\ListViewDialog.h +# End Source File +# Begin Source File + SOURCE=..\FileManager\OverwriteDialog.cpp # End Source File # Begin Source File @@ -975,6 +991,14 @@ SOURCE=..\..\..\Windows\Control\Static.h # End Group # Begin Source File +SOURCE=..\..\..\Windows\Clipboard.cpp +# End Source File +# Begin Source File + +SOURCE=..\..\..\Windows\Clipboard.h +# End Source File +# Begin Source File + SOURCE=..\..\..\Windows\COM.h # End Source File # Begin Source File @@ -1051,6 +1075,14 @@ SOURCE=..\..\..\Windows\FileSystem.h # End Source File # Begin Source File +SOURCE=..\..\..\Windows\MemoryGlobal.cpp +# End Source File +# Begin Source File + +SOURCE=..\..\..\Windows\MemoryGlobal.h +# End Source File +# Begin Source File + SOURCE=..\..\..\Windows\MemoryLock.cpp # End Source File # Begin Source File diff --git a/CPP/7zip/UI/GUI/HashGUI.cpp b/CPP/7zip/UI/GUI/HashGUI.cpp index 621bd0ab..5aef24c1 100644 --- a/CPP/7zip/UI/GUI/HashGUI.cpp +++ b/CPP/7zip/UI/GUI/HashGUI.cpp @@ -9,6 +9,7 @@ #include "../FileManager/FormatUtils.h" #include "../FileManager/LangUtils.h" +#include "../FileManager/ListViewDialog.h" #include "../FileManager/OverwriteDialogRes.h" #include "../FileManager/ProgressDialog2.h" #include "../FileManager/ProgressDialog2Res.h" @@ -19,13 +20,18 @@ using namespace NWindows; + + class CHashCallbackGUI: public CProgressThreadVirt, public IHashCallbackUI { UInt64 NumFiles; bool _curIsFolder; UString FirstFileName; + CPropNameValPairs PropNameValPairs; + HRESULT ProcessVirt(); + virtual void ProcessWasFinished_GuiVirt(); public: const NWildcard::CCensor *censor; @@ -40,49 +46,62 @@ public: void AddErrorMessage(DWORD systemError, const wchar_t *name) { - ProgressDialog.Sync.AddError_Code_Name(systemError, name); + Sync.AddError_Code_Name(systemError, name); } }; -static void AddValuePair(UString &s, UINT resourceID, UInt64 value) + +void AddValuePair(CPropNameValPairs &pairs, UINT resourceID, UInt64 value) { - AddLangString(s, resourceID); - s += ": "; + CProperty &pair = pairs.AddNew(); + AddLangString(pair.Name, resourceID); char sz[32]; ConvertUInt64ToString(value, sz); - s += sz; - s.Add_LF(); + pair.Value = sz; } -static void AddSizeValuePair(UString &s, UINT resourceID, UInt64 value) + +void AddSizeValue(UString &s, UInt64 value) { - AddLangString(s, resourceID); - s += ": "; { wchar_t sz[32]; ConvertUInt64ToString(value, sz); s += MyFormatNew(IDS_FILE_SIZE, sz); } + if (value >= (1 << 10)) { + char c; + if (value >= ((UInt64)10 << 30)) { value >>= 30; c = 'G'; } + if (value >= (10 << 20)) { value >>= 20; c = 'M'; } + else { value >>= 10; c = 'K'; } char sz[32]; - ConvertUInt64ToString(value >> 20, sz); + ConvertUInt64ToString(value, sz); s += " ("; s += sz; - s += " MB)"; - s.Add_LF(); + s += " "; + s += (wchar_t)c; + s += "iB)"; } } +void AddSizeValuePair(CPropNameValPairs &pairs, UINT resourceID, UInt64 value) +{ + CProperty &pair = pairs.AddNew(); + LangString(resourceID, pair.Name); + AddSizeValue(pair.Value, value); +} + + HRESULT CHashCallbackGUI::StartScanning() { - CProgressSync &sync = ProgressDialog.Sync; + CProgressSync &sync = Sync; sync.Set_Status(LangString(IDS_SCANNING)); return CheckBreak(); } HRESULT CHashCallbackGUI::ScanProgress(const CDirItemsStat &st, const FString &path, bool isDir) { - return ProgressDialog.Sync.ScanProgress(st.NumFiles, st.GetTotalBytes(), path, isDir); + return Sync.ScanProgress(st.NumFiles, st.GetTotalBytes(), path, isDir); } HRESULT CHashCallbackGUI::ScanError(const FString &path, DWORD systemError) @@ -98,26 +117,26 @@ HRESULT CHashCallbackGUI::FinishScanning(const CDirItemsStat &st) HRESULT CHashCallbackGUI::CheckBreak() { - return ProgressDialog.Sync.CheckStop(); + return Sync.CheckStop(); } HRESULT CHashCallbackGUI::SetNumFiles(UInt64 numFiles) { - CProgressSync &sync = ProgressDialog.Sync; + CProgressSync &sync = Sync; sync.Set_NumFilesTotal(numFiles); return CheckBreak(); } HRESULT CHashCallbackGUI::SetTotal(UInt64 size) { - CProgressSync &sync = ProgressDialog.Sync; + CProgressSync &sync = Sync; sync.Set_NumBytesTotal(size); return CheckBreak(); } HRESULT CHashCallbackGUI::SetCompleted(const UInt64 *completed) { - return ProgressDialog.Sync.Set_NumBytesCur(completed); + return Sync.Set_NumBytesCur(completed); } HRESULT CHashCallbackGUI::BeforeFirstFile(const CHashBundle & /* hb */) @@ -130,7 +149,7 @@ HRESULT CHashCallbackGUI::GetStream(const wchar_t *name, bool isFolder) if (NumFiles == 0) FirstFileName = name; _curIsFolder = isFolder; - CProgressSync &sync = ProgressDialog.Sync; + CProgressSync &sync = Sync; sync.Set_FilePath(name, isFolder); return CheckBreak(); } @@ -147,49 +166,47 @@ HRESULT CHashCallbackGUI::OpenFileError(const FString &path, DWORD systemError) HRESULT CHashCallbackGUI::SetOperationResult(UInt64 /* fileSize */, const CHashBundle & /* hb */, bool /* showHash */) { - CProgressSync &sync = ProgressDialog.Sync; + CProgressSync &sync = Sync; if (!_curIsFolder) NumFiles++; sync.Set_NumFilesCur(NumFiles); return CheckBreak(); } -static void AddHashString(UString &s, const CHasherState &h, unsigned digestIndex, const wchar_t *title) +static void AddHashString(CProperty &s, const CHasherState &h, unsigned digestIndex) { - s += title; - s.Add_Space(); char temp[k_HashCalc_DigestSize_Max * 2 + 4]; AddHashHexToString(temp, h.Digests[digestIndex], h.DigestSize); - s += temp; - s.Add_LF(); + s.Value = temp; } -static void AddHashResString(UString &s, const CHasherState &h, unsigned digestIndex, UInt32 resID) +static void AddHashResString(CPropNameValPairs &s, const CHasherState &h, unsigned digestIndex, UInt32 resID) { - UString s2 = LangString(resID); + CProperty &pair = s.AddNew(); + UString &s2 = pair.Name; + LangString(resID, s2); UString name (h.Name); s2.Replace(L"CRC", name); - AddHashString(s, h, digestIndex, s2); + s2.Replace(L":", L""); + AddHashString(pair, h, digestIndex); } -void AddHashBundleRes(UString &s, const CHashBundle &hb, const UString &firstFileName) + +void AddHashBundleRes(CPropNameValPairs &s, const CHashBundle &hb, const UString &firstFileName) { if (hb.NumErrors != 0) - { AddValuePair(s, IDS_PROP_NUM_ERRORS, hb.NumErrors); - s.Add_LF(); - } if (hb.NumFiles == 1 && hb.NumDirs == 0 && !firstFileName.IsEmpty()) { - AddLangString(s, IDS_PROP_NAME); - s += ": "; - s += firstFileName; - s.Add_LF(); + CProperty &pair = s.AddNew(); + LangString(IDS_PROP_NAME, pair.Name); + pair.Value = firstFileName; } else { - AddValuePair(s, IDS_PROP_FOLDERS, hb.NumDirs); + if (hb.NumDirs != 0) + AddValuePair(s, IDS_PROP_FOLDERS, hb.NumDirs); AddValuePair(s, IDS_PROP_FILES, hb.NumFiles); } @@ -197,25 +214,18 @@ void AddHashBundleRes(UString &s, const CHashBundle &hb, const UString &firstFil if (hb.NumAltStreams != 0) { - s.Add_LF(); AddValuePair(s, IDS_PROP_NUM_ALT_STREAMS, hb.NumAltStreams); AddSizeValuePair(s, IDS_PROP_ALT_STREAMS_SIZE, hb.AltStreamsSize); } - if (hb.NumErrors == 0 && hb.Hashers.IsEmpty()) - { - s.Add_LF(); - AddLangString(s, IDS_MESSAGE_NO_ERRORS); - } - FOR_VECTOR (i, hb.Hashers) { - s.Add_LF(); const CHasherState &h = hb.Hashers[i]; if (hb.NumFiles == 1 && hb.NumDirs == 0) { - s += h.Name; - AddHashString(s, h, k_HashCalc_Index_DataSum, L":"); + CProperty &pair = s.AddNew(); + pair.Name += h.Name; + AddHashString(pair, h, k_HashCalc_Index_DataSum); } else { @@ -229,32 +239,55 @@ void AddHashBundleRes(UString &s, const CHashBundle &hb, const UString &firstFil } } + +void AddHashBundleRes(UString &s, const CHashBundle &hb, const UString &firstFileName) +{ + CPropNameValPairs pairs; + AddHashBundleRes(pairs, hb, firstFileName); + + FOR_VECTOR (i, pairs) + { + const CProperty &pair = pairs[i]; + s += pair.Name; + s += ": "; + s += pair.Value; + s.Add_LF(); + } + + if (hb.NumErrors == 0 && hb.Hashers.IsEmpty()) + { + s.Add_LF(); + AddLangString(s, IDS_MESSAGE_NO_ERRORS); + s.Add_LF(); + } +} + + HRESULT CHashCallbackGUI::AfterLastFile(const CHashBundle &hb) { - UString s; - AddHashBundleRes(s, hb, FirstFileName); + AddHashBundleRes(PropNameValPairs, hb, FirstFileName); - CProgressSync &sync = ProgressDialog.Sync; + CProgressSync &sync = Sync; sync.Set_NumFilesCur(hb.NumFiles); - CProgressMessageBoxPair &pair = GetMessagePair(hb.NumErrors != 0); - pair.Message = s; - LangString(IDS_CHECKSUM_INFORMATION, pair.Title); + // CProgressMessageBoxPair &pair = GetMessagePair(hb.NumErrors != 0); + // pair.Message = s; + // LangString(IDS_CHECKSUM_INFORMATION, pair.Title); return S_OK; } + HRESULT CHashCallbackGUI::ProcessVirt() { NumFiles = 0; - AString errorInfo; HRESULT res = HashCalc(EXTERNAL_CODECS_LOC_VARS *censor, *options, errorInfo, this); - return res; } + HRESULT HashCalcGUI( DECL_EXTERNAL_CODECS_LOC_VARS const NWildcard::CCensor &censor, @@ -268,15 +301,49 @@ HRESULT HashCalcGUI( t.censor = &censor; t.options = &options; - t.ProgressDialog.ShowCompressionInfo = false; + t.ShowCompressionInfo = false; const UString title = LangString(IDS_CHECKSUM_CALCULATING); - t.ProgressDialog.MainTitle = "7-Zip ZS"; // LangString(IDS_APP_TITLE); - t.ProgressDialog.MainAddTitle = title; - t.ProgressDialog.MainAddTitle.Add_Space(); + t.MainTitle = "7-Zip ZS"; // LangString(IDS_APP_TITLE); + t.MainAddTitle = title; + t.MainAddTitle.Add_Space(); RINOK(t.Create(title)); - messageWasDisplayed = t.ThreadFinishedOK && t.ProgressDialog.MessagesDisplayed; + messageWasDisplayed = t.ThreadFinishedOK && t.MessagesDisplayed; return S_OK; } + + +void ShowHashResults(const CPropNameValPairs &propPairs, HWND hwnd) +{ + CListViewDialog lv; + + FOR_VECTOR (i, propPairs) + { + const CProperty &pair = propPairs[i]; + lv.Strings.Add(pair.Name); + lv.Values.Add(pair.Value); + } + + lv.Title = LangString(IDS_CHECKSUM_INFORMATION); + lv.DeleteIsAllowed = true; + lv.SelectFirst = false; + lv.NumColumns = 2; + + lv.Create(hwnd); +} + + +void ShowHashResults(const CHashBundle &hb, const UString &firstFileName, HWND hwnd) +{ + CPropNameValPairs propPairs; + AddHashBundleRes(propPairs, hb, firstFileName); + ShowHashResults(propPairs, hwnd); +} + + +void CHashCallbackGUI::ProcessWasFinished_GuiVirt() +{ + ShowHashResults(PropNameValPairs, *this); +} diff --git a/CPP/7zip/UI/GUI/HashGUI.h b/CPP/7zip/UI/GUI/HashGUI.h index 4fb0666c..d6caa53e 100644 --- a/CPP/7zip/UI/GUI/HashGUI.h +++ b/CPP/7zip/UI/GUI/HashGUI.h @@ -4,6 +4,7 @@ #define __HASH_GUI_H #include "../Common/HashCalc.h" +#include "../Common/Property.h" HRESULT HashCalcGUI( DECL_EXTERNAL_CODECS_LOC_VARS @@ -11,6 +12,16 @@ HRESULT HashCalcGUI( const CHashOptions &options, bool &messageWasDisplayed); +typedef CObjectVector CPropNameValPairs; + +void AddValuePair(CPropNameValPairs &pairs, UINT resourceID, UInt64 value); +void AddSizeValue(UString &s, UInt64 value); +void AddSizeValuePair(CPropNameValPairs &pairs, UINT resourceID, UInt64 value); + +void AddHashBundleRes(CPropNameValPairs &s, const CHashBundle &hb, const UString &firstFileName); void AddHashBundleRes(UString &s, const CHashBundle &hb, const UString &firstFileName); +void ShowHashResults(const CPropNameValPairs &propPairs, HWND hwnd); +void ShowHashResults(const CHashBundle &hb, const UString &firstFileName, HWND hwnd); + #endif diff --git a/CPP/7zip/UI/GUI/UpdateGUI.cpp b/CPP/7zip/UI/GUI/UpdateGUI.cpp index b3f09ecc..280ed78a 100644 --- a/CPP/7zip/UI/GUI/UpdateGUI.cpp +++ b/CPP/7zip/UI/GUI/UpdateGUI.cpp @@ -460,7 +460,7 @@ HRESULT UpdateGUI( tu.cmdArcPath = &cmdArcPath; tu.UpdateCallbackGUI = callback; - tu.UpdateCallbackGUI->ProgressDialog = &tu.ProgressDialog; + tu.UpdateCallbackGUI->ProgressDialog = &tu; tu.UpdateCallbackGUI->Init(); UString title = LangString(IDS_PROGRESS_COMPRESSING); @@ -476,10 +476,10 @@ HRESULT UpdateGUI( tu.WildcardCensor = &censor; tu.Options = &options; - tu.ProgressDialog.IconID = IDI_ICON; + tu.IconID = IDI_ICON; RINOK(tu.Create(title, hwndParent)); - messageWasDisplayed = tu.ThreadFinishedOK && tu.ProgressDialog.MessagesDisplayed; + messageWasDisplayed = tu.ThreadFinishedOK && tu.MessagesDisplayed; return tu.Result; } diff --git a/CPP/7zip/UI/GUI/makefile b/CPP/7zip/UI/GUI/makefile index 0a51c8e8..088da0aa 100644 --- a/CPP/7zip/UI/GUI/makefile +++ b/CPP/7zip/UI/GUI/makefile @@ -36,6 +36,7 @@ COMMON_OBJS = \ $O\Wildcard.obj \ WIN_OBJS = \ + $O\Clipboard.obj \ $O\CommonDialog.obj \ $O\DLL.obj \ $O\ErrorMsg.obj \ @@ -45,6 +46,7 @@ WIN_OBJS = \ $O\FileLink.obj \ $O\FileName.obj \ $O\FileSystem.obj \ + $O\MemoryGlobal.obj \ $O\MemoryLock.obj \ $O\PropVariant.obj \ $O\PropVariantConv.obj \ @@ -102,10 +104,12 @@ AR_COMMON_OBJS = \ $O\OutStreamWithCRC.obj \ FM_OBJS = \ + $O\EditDialog.obj \ $O\ExtractCallback.obj \ $O\FormatUtils.obj \ $O\HelpUtils.obj \ $O\LangUtils.obj \ + $O\ListViewDialog.obj \ $O\OpenCallback.obj \ $O\ProgramLocation.obj \ $O\PropertyName.obj \ diff --git a/CPP/7zip/UI/GUI/resource.rc b/CPP/7zip/UI/GUI/resource.rc index 4dae3c3d..03a9f419 100644 --- a/CPP/7zip/UI/GUI/resource.rc +++ b/CPP/7zip/UI/GUI/resource.rc @@ -21,3 +21,5 @@ IDI_ICON ICON "FM.ico" #include "Extract.rc" #include "../FileManager/BrowseDialog.rc" #include "../FileManager/ComboDialog.rc" +#include "../FileManager/EditDialog.rc" +#include "../FileManager/ListViewDialog.rc" diff --git a/CPP/Build.mak b/CPP/Build.mak index c090f758..b1c1c569 100644 --- a/CPP/Build.mak +++ b/CPP/Build.mak @@ -13,11 +13,11 @@ O=O !ENDIF !IF "$(CPU)" == "AMD64" -MY_ML = ml64 -Dx64 +MY_ML = ml64 -Dx64 -WX !ELSEIF "$(CPU)" == "ARM" -MY_ML = armasm +MY_ML = armasm -WX !ELSE -MY_ML = ml +MY_ML = ml -WX !ENDIF @@ -30,9 +30,9 @@ LFLAGS = $(LFLAGS) /ENTRY:mainACRTStartup !IFNDEF NEW_COMPILER LFLAGS = $(LFLAGS) !ENDIF -# !IF "$(CPU)" != "ARM" +!IF "$(CPU)" != "ARM" && "$(CPU)" != "ARM64" CFLAGS = $(CFLAGS) -Gr -# !ENDIF +!ENDIF LIBS = $(LIBS) user32.lib advapi32.lib shell32.lib !ENDIF @@ -80,9 +80,11 @@ LFLAGS = $(LFLAGS) /LTCG /LARGEADDRESSAWARE !IFDEF DEF_FILE LFLAGS = $(LFLAGS) -DLL -DEF:$(DEF_FILE) !ELSE -# !IF "$(CPU)" != "ARM" +!IF defined(MY_FIXED) && "$(CPU)" != "ARM" && "$(CPU)" != "ARM64" LFLAGS = $(LFLAGS) /FIXED -# !ENDIF +!ELSE +LFLAGS = $(LFLAGS) /FIXED:NO +!ENDIF # /BASE:0x400000 !ENDIF @@ -113,10 +115,11 @@ COMPLB = $(CC) $(CFLAGS_O1) -Yu"StdAfx.h" -Fp$O/a.pch $< # COMPLB_O2 = $(CC) $(CFLAGS_O2) -Yu"StdAfx.h" -Fp$O/a.pch $< COMPLB_O2 = $(CC) $(CFLAGS_O2) $< -CCOMPL_PCH = $(CC) $(CFLAGS_O2) -Yc"Precomp.h" -Fp$O/a.pch $** -CCOMPL_USE = $(CC) $(CFLAGS_O2) -Yu"Precomp.h" -Fp$O/a.pch $** -CCOMPL = $(CC) $(CFLAGS_O2) $** -CCOMPLB = $(CC) $(CFLAGS_O2) $< +CFLAGS_C_ALL = $(CFLAGS_O2) $(CFLAGS_C_SPEC) +CCOMPL_PCH = $(CC) $(CFLAGS_C_ALL) -Yc"Precomp.h" -Fp$O/a.pch $** +CCOMPL_USE = $(CC) $(CFLAGS_C_ALL) -Yu"Precomp.h" -Fp$O/a.pch $** +CCOMPL = $(CC) $(CFLAGS_C_ALL) $** +CCOMPLB = $(CC) $(CFLAGS_C_ALL) $< all: $(PROGPATH) diff --git a/CPP/Common/MyString.h b/CPP/Common/MyString.h index 9e8f8a2a..f484ad22 100644 --- a/CPP/Common/MyString.h +++ b/CPP/Common/MyString.h @@ -545,6 +545,8 @@ public: void ReplaceOneCharAtPos(unsigned pos, wchar_t c) { _chars[pos] = c; } + wchar_t *GetBuf() { return _chars; } + wchar_t *GetBuf(unsigned minLen) { if (minLen > _limit) @@ -571,7 +573,7 @@ public: } UString &operator=(wchar_t c); - UString &operator=(char c) { return (*this)=((wchar_t)c); } + UString &operator=(char c) { return (*this)=((wchar_t)(unsigned char)c); } UString &operator=(const wchar_t *s); UString &operator=(const UString &s); void SetFrom(const wchar_t *s, unsigned len); // no check diff --git a/CPP/Common/MyWindows.h b/CPP/Common/MyWindows.h index 5f04c691..cc784781 100644 --- a/CPP/Common/MyWindows.h +++ b/CPP/Common/MyWindows.h @@ -16,6 +16,7 @@ #include // for wchar_t #include +// #include // for uintptr_t #include "MyGuidDef.h" @@ -53,6 +54,8 @@ typedef long BOOL; // typedef size_t ULONG_PTR; typedef size_t DWORD_PTR; +// typedef uintptr_t UINT_PTR; +// typedef ptrdiff_t UINT_PTR; typedef Int64 LONGLONG; typedef UInt64 ULONGLONG; @@ -81,6 +84,8 @@ typedef struct _FILETIME typedef ULONG PROPID; typedef LONG SCODE; +#define ERROR_NEGATIVE_SEEK 131L + #define S_OK ((HRESULT)0x00000000L) #define S_FALSE ((HRESULT)0x00000001L) #define E_NOTIMPL ((HRESULT)0x80004001L) diff --git a/CPP/Common/StdOutStream.cpp b/CPP/Common/StdOutStream.cpp index e50c228e..dc6d4bd4 100644 --- a/CPP/Common/StdOutStream.cpp +++ b/CPP/Common/StdOutStream.cpp @@ -75,6 +75,65 @@ void CStdOutStream::PrintUString(const UString &s, AString &temp) *this << (const char *)temp; } + +static const wchar_t kReplaceChar = '_'; + +void CStdOutStream::Normalize_UString__LF_Allowed(UString &s) +{ + unsigned len = s.Len(); + wchar_t *d = s.GetBuf(); + + if (IsTerminalMode) + for (unsigned i = 0; i < len; i++) + { + wchar_t c = d[i]; + if (c <= 13 && c >= 7 && c != '\n') + d[i] = kReplaceChar; + } +} + +void CStdOutStream::Normalize_UString(UString &s) +{ + unsigned len = s.Len(); + wchar_t *d = s.GetBuf(); + + if (IsTerminalMode) + for (unsigned i = 0; i < len; i++) + { + wchar_t c = d[i]; + if (c <= 13 && c >= 7) + d[i] = kReplaceChar; + } + else + for (unsigned i = 0; i < len; i++) + { + wchar_t c = d[i]; + if (c == '\n') + d[i] = kReplaceChar; + } +} + +void CStdOutStream::NormalizePrint_UString(const UString &s, UString &tempU, AString &tempA) +{ + tempU = s; + Normalize_UString(tempU); + PrintUString(tempU, tempA); +} + +void CStdOutStream::NormalizePrint_UString(const UString &s) +{ + NormalizePrint_wstr(s); +} + +void CStdOutStream::NormalizePrint_wstr(const wchar_t *s) +{ + UString tempU = s; + Normalize_UString(tempU); + AString tempA; + PrintUString(tempU, tempA); +} + + CStdOutStream & CStdOutStream::operator<<(Int32 number) throw() { char s[32]; diff --git a/CPP/Common/StdOutStream.h b/CPP/Common/StdOutStream.h index 45738b26..475954c0 100644 --- a/CPP/Common/StdOutStream.h +++ b/CPP/Common/StdOutStream.h @@ -13,7 +13,9 @@ class CStdOutStream FILE *_stream; bool _streamIsOpen; public: - CStdOutStream(): _stream(0), _streamIsOpen(false) {}; + bool IsTerminalMode; + + CStdOutStream(): _stream(0), _streamIsOpen(false), IsTerminalMode(false) {}; CStdOutStream(FILE *stream): _stream(stream), _streamIsOpen(false) {}; ~CStdOutStream() { Close(); } @@ -50,6 +52,13 @@ public: CStdOutStream & operator<<(const wchar_t *s); void PrintUString(const UString &s, AString &temp); + + void Normalize_UString__LF_Allowed(UString &s); + void Normalize_UString(UString &s); + + void NormalizePrint_UString(const UString &s, UString &tempU, AString &tempA); + void NormalizePrint_UString(const UString &s); + void NormalizePrint_wstr(const wchar_t *s); }; CStdOutStream & endl(CStdOutStream & outStream) throw(); diff --git a/CPP/Windows/Control/Dialog.cpp b/CPP/Windows/Control/Dialog.cpp index c8a09ed2..8e61a2b2 100644 --- a/CPP/Windows/Control/Dialog.cpp +++ b/CPP/Windows/Control/Dialog.cpp @@ -72,6 +72,7 @@ bool CDialog::OnButtonClicked(int buttonID, HWND /* buttonHWND */) { case IDOK: OnOK(); break; case IDCANCEL: OnCancel(); break; + case IDCLOSE: OnClose(); break; case IDHELP: OnHelp(); break; default: return false; } diff --git a/CPP/Windows/Control/Dialog.h b/CPP/Windows/Control/Dialog.h index b160958d..f9c3442f 100644 --- a/CPP/Windows/Control/Dialog.h +++ b/CPP/Windows/Control/Dialog.h @@ -105,6 +105,7 @@ public: virtual bool OnButtonClicked(int buttonID, HWND buttonHWND); virtual void OnOK() {}; virtual void OnCancel() {}; + virtual void OnClose() {} virtual bool OnNotify(UINT /* controlID */, LPNMHDR /* lParam */) { return false; } virtual bool OnTimer(WPARAM /* timerID */, LPARAM /* callback */) { return false; } @@ -133,6 +134,7 @@ public: #endif virtual void OnOK() { Destroy(); } virtual void OnCancel() { Destroy(); } + virtual void OnClose() { Destroy(); } }; class CModalDialog: public CDialog @@ -147,6 +149,7 @@ public: bool End(INT_PTR result) { return BOOLToBool(::EndDialog(_window, result)); } virtual void OnOK() { End(IDOK); } virtual void OnCancel() { End(IDCANCEL); } + virtual void OnClose() { End(IDCLOSE); } }; class CDialogChildControl: public NWindows::CWindow diff --git a/CPP/Windows/FileDir.cpp b/CPP/Windows/FileDir.cpp index 91808b6d..124569e3 100644 --- a/CPP/Windows/FileDir.cpp +++ b/CPP/Windows/FileDir.cpp @@ -667,12 +667,27 @@ bool CTempFile::Remove() bool CTempFile::MoveTo(CFSTR name, bool deleteDestBefore) { + // DWORD attrib = 0; if (deleteDestBefore) + { if (NFind::DoesFileExist(name)) + { + // attrib = NFind::GetFileAttrib(name); if (!DeleteFileAlways(name)) return false; + } + } DisableDeleting(); return MyMoveFile(_path, name); + + /* + if (attrib != INVALID_FILE_ATTRIBUTES && (attrib & FILE_ATTRIBUTE_READONLY)) + { + DWORD attrib2 = NFind::GetFileAttrib(name); + if (attrib2 != INVALID_FILE_ATTRIBUTES) + SetFileAttrib(name, attrib2 | FILE_ATTRIBUTE_READONLY); + } + */ } bool CTempDir::Create(CFSTR prefix) diff --git a/CPP/Windows/FileIO.h b/CPP/Windows/FileIO.h index af4785f6..e31bc20c 100644 --- a/CPP/Windows/FileIO.h +++ b/CPP/Windows/FileIO.h @@ -45,7 +45,11 @@ struct CReparseAttr UString PrintName; CReparseAttr(): Tag(0), Flags(0) {} - bool Parse(const Byte *p, size_t size); + + // Parse() + // returns true and (errorCode = 0), if (correct MOUNT_POINT or SYMLINK) + // returns false and (errorCode = ERROR_REPARSE_TAG_MISMATCH), if not (MOUNT_POINT or SYMLINK) + bool Parse(const Byte *p, size_t size, DWORD &errorCode); bool IsMountPoint() const { return Tag == _my_IO_REPARSE_TAG_MOUNT_POINT; } // it's Junction bool IsSymLink() const { return Tag == _my_IO_REPARSE_TAG_SYMLINK; } @@ -171,7 +175,12 @@ public: bool OpenReparse(CFSTR fileName) { - return Open(fileName, FILE_SHARE_READ, OPEN_EXISTING, + // 17.02 fix: to support Windows XP compatibility junctions: + // we use Create() with (desiredAccess = 0) instead of Open() with GENERIC_READ + return + Create(fileName, 0, + // Open(fileName, + FILE_SHARE_READ, OPEN_EXISTING, FILE_FLAG_OPEN_REPARSE_POINT | FILE_FLAG_BACKUP_SEMANTICS); } diff --git a/CPP/Windows/FileLink.cpp b/CPP/Windows/FileLink.cpp index cef73120..b5e47e73 100644 --- a/CPP/Windows/FileLink.cpp +++ b/CPP/Windows/FileLink.cpp @@ -194,8 +194,9 @@ static void GetString(const Byte *p, unsigned len, UString &res) res.ReleaseBuf_SetLen(i); } -bool CReparseAttr::Parse(const Byte *p, size_t size) +bool CReparseAttr::Parse(const Byte *p, size_t size, DWORD &errorCode) { + errorCode = ERROR_INVALID_REPARSE_DATA; if (size < 8) return false; Tag = Get32(p); @@ -209,8 +210,10 @@ bool CReparseAttr::Parse(const Byte *p, size_t size) */ if (Tag != _my_IO_REPARSE_TAG_MOUNT_POINT && Tag != _my_IO_REPARSE_TAG_SYMLINK) - // return true; + { + errorCode = ERROR_REPARSE_TAG_MISMATCH; // ERROR_REPARSE_TAG_INVALID return false; + } if (Get16(p + 6) != 0) // padding return false; @@ -247,6 +250,7 @@ bool CReparseAttr::Parse(const Byte *p, size_t size) GetString(p + subOffs, subLen >> 1, SubsName); GetString(p + printOffs, printLen >> 1, PrintName); + errorCode = 0; return true; } diff --git a/CPP/Windows/MemoryLock.cpp b/CPP/Windows/MemoryLock.cpp index 7a2cfd57..3cd82e50 100644 --- a/CPP/Windows/MemoryLock.cpp +++ b/CPP/Windows/MemoryLock.cpp @@ -67,6 +67,31 @@ bool EnablePrivilege(LPCTSTR privilegeName, bool enable) return res; } + + +typedef void (WINAPI * Func_RtlGetVersion) (OSVERSIONINFOEXW *); + +/* + We suppose that Window 10 works incorrectly with "Large Pages" at: + - Windows 10 1703 (15063) + - Windows 10 1709 (16299) +*/ + +unsigned Get_LargePages_RiskLevel() +{ + OSVERSIONINFOEXW vi; + HMODULE ntdll = ::GetModuleHandleW(L"ntdll.dll"); + if (!ntdll) + return 0; + Func_RtlGetVersion func = (Func_RtlGetVersion)GetProcAddress(ntdll, "RtlGetVersion"); + if (!func) + return 0; + func(&vi); + return (vi.dwPlatformId == VER_PLATFORM_WIN32_NT + && vi.dwMajorVersion + vi.dwMinorVersion == 10 + && vi.dwBuildNumber <= 16299) ? 1 : 0; +} + #endif }} diff --git a/CPP/Windows/MemoryLock.h b/CPP/Windows/MemoryLock.h index 1336fe55..d82910fe 100644 --- a/CPP/Windows/MemoryLock.h +++ b/CPP/Windows/MemoryLock.h @@ -31,6 +31,8 @@ inline void EnablePrivilege_SymLink() // Do we need to set SE_BACKUP_NAME ? } +unsigned Get_LargePages_RiskLevel(); + #endif }} diff --git a/CPP/Windows/Registry.cpp b/CPP/Windows/Registry.cpp index 71ca17f0..01466214 100644 --- a/CPP/Windows/Registry.cpp +++ b/CPP/Windows/Registry.cpp @@ -152,7 +152,7 @@ LONG CKey::SetValue(LPCWSTR name, LPCWSTR value) MYASSERT(value != NULL); MYASSERT(_object != NULL); if (g_IsNT) - return RegSetValueExW(_object, name, NULL, REG_SZ, + return RegSetValueExW(_object, name, 0, REG_SZ, (const BYTE * )value, (DWORD)((wcslen(value) + 1) * sizeof(wchar_t))); return SetValue(name == 0 ? 0 : (LPCSTR)GetSystemString(name), value == 0 ? 0 : (LPCSTR)GetSystemString(value)); diff --git a/DOC/7zip.inf b/DOC/7zip.inf index 2c5311b2..4f79b44c 100644 --- a/DOC/7zip.inf +++ b/DOC/7zip.inf @@ -10,8 +10,8 @@ AppName = "7-Zip" InstallDir = %CE1%\%AppName% [Strings] -AppVer = "17.01" -AppDate = "2017-08-28" +AppVer = "18.05" +AppDate = "2018-04-30" [CEDevice] ; ProcessorType = 2577 ; ARM diff --git a/DOC/7zip.nsi b/DOC/7zip.nsi index f8e37859..ae55c49c 100644 --- a/DOC/7zip.nsi +++ b/DOC/7zip.nsi @@ -1,8 +1,8 @@ ;-------------------------------- ;Defines -!define VERSION_MAJOR 17 -!define VERSION_MINOR 01 +!define VERSION_MAJOR 18 +!define VERSION_MINOR 05 !define VERSION_POSTFIX_FULL " ZS" !ifdef WIN64 !ifdef IA64 @@ -220,6 +220,7 @@ Section File ja.txt File ka.txt File kaa.txt + File kab.txt File kk.txt File ko.txt File ku.txt @@ -421,6 +422,7 @@ Section Uninstall Delete $INSTDIR\Lang\ja.txt Delete $INSTDIR\Lang\ka.txt Delete $INSTDIR\Lang\kaa.txt + Delete $INSTDIR\Lang\kab.txt Delete $INSTDIR\Lang\kk.txt Delete $INSTDIR\Lang\ko.txt Delete $INSTDIR\Lang\ku.txt diff --git a/DOC/7zip.wxs b/DOC/7zip.wxs index bfc7e709..f97502d5 100644 --- a/DOC/7zip.wxs +++ b/DOC/7zip.wxs @@ -1,7 +1,7 @@ - - + + @@ -104,6 +104,8 @@ + + @@ -294,6 +296,7 @@ + diff --git a/DOC/License.txt b/DOC/License.txt index c31f4db5..cae06938 100644 --- a/DOC/License.txt +++ b/DOC/License.txt @@ -3,15 +3,20 @@ License for use and distribution ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - 7-Zip Copyright (C) 1999-2017 Igor Pavlov. + 7-Zip Copyright (C) 1999-2018 Igor Pavlov. - Licenses for files are: + The licenses for files are: + + 1) CPP/7zip/Compress/Rar* files: the "GNU LGPL" with "unRAR license restriction" + 2) CPP/7zip/Compress/LzfseDecoder.cpp: the "BSD 3-clause License" + 3) Some files are "public domain" files, if "public domain" status is stated in source file. + 4) the "GNU LGPL" for all other files. If there is no license information in + some source file, that file is under the "GNU LGPL". + + The "GNU LGPL" with "unRAR license restriction" means that you must follow both + "GNU LGPL" rules and "unRAR license restriction" rules. - 1) CPP/7zip/Compress/Rar* files: GNU LGPL + unRAR restriction - 2) All other files: GNU LGPL - The GNU LGPL + unRAR restriction means that you must follow both - GNU LGPL rules and unRAR restriction rules. GNU LGPL information @@ -32,8 +37,41 @@ Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA - unRAR restriction - ----------------- + + + BSD 3-clause License + -------------------- + + The "BSD 3-clause License" is used for the code in LzfseDecoder.cpp that implements LZFSE data decompression. + That code was derived from the code in the "LZFSE compression library" developed by Apple Inc, + that also uses the "BSD 3-clause License": + + ---- + Copyright (c) 2015-2016, Apple Inc. All rights reserved. + + Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: + + 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. + + 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer + in the documentation and/or other materials provided with the distribution. + + 3. Neither the name of the copyright holder(s) nor the names of any contributors may be used to endorse or promote products derived + from this software without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE + COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + ---- + + + + + unRAR license restriction + ------------------------- The decompression engine for RAR archives was developed using source code of unRAR program. diff --git a/DOC/lzma.txt b/DOC/lzma.txt index f9a80aeb..1f92142e 100644 --- a/DOC/lzma.txt +++ b/DOC/lzma.txt @@ -33,7 +33,7 @@ If you want to use old interfaces you can download previous version of LZMA SDK from sourceforge.net site. To use ANSI-C LZMA Decoder you need the following files: -1) LzmaDec.h + LzmaDec.c + Types.h +1) LzmaDec.h + LzmaDec.c + 7zTypes.h + Precomp.h + Compiler.h Look example code: C/Util/Lzma/LzmaUtil.c @@ -70,7 +70,7 @@ You can use p = p; operator to disable compiler warnings. Single-call Decompressing ------------------------- When to use: RAM->RAM decompressing -Compile files: LzmaDec.h + LzmaDec.c + Types.h +Compile files: LzmaDec.h + LzmaDec.c + 7zTypes.h Compile defines: no defines Memory Requirements: - Input buffer: compressed size @@ -124,7 +124,7 @@ Multi-call State Decompressing (zlib-like interface) ---------------------------------------------------- When to use: file->file decompressing -Compile files: LzmaDec.h + LzmaDec.c + Types.h +Compile files: LzmaDec.h + LzmaDec.c + 7zTypes.h Memory Requirements: - Buffer for input stream: any size (for example, 16 KB) @@ -167,7 +167,7 @@ How To compress data -------------------- Compile files: - Types.h + 7zTypes.h Threads.h LzmaEnc.h LzmaEnc.c diff --git a/DOC/readme.txt b/DOC/readme.txt index f45ba7d2..3118bdeb 100644 --- a/DOC/readme.txt +++ b/DOC/readme.txt @@ -1,9 +1,9 @@ -7-Zip 17.00 beta Sources ------------------------- +7-Zip 18.05 Sources +------------------- 7-Zip is a file archiver for Windows. -7-Zip Copyright (C) 1999-2017 Igor Pavlov. +7-Zip Copyright (C) 1999-2018 Igor Pavlov. License Info diff --git a/DOC/src-history.txt b/DOC/src-history.txt index b4d3383b..57db3785 100644 --- a/DOC/src-history.txt +++ b/DOC/src-history.txt @@ -1,6 +1,27 @@ HISTORY of the 7-Zip source code -------------------------------- +18.05 2018-04-30 +------------------------- +- The speed for LZMA/LZMA2 compressing was increased + by 8% for fastest/fast compression levels and + by 3% for normal/maximum compression levels. +- Previous versions of 7-Zip could work incorrectly in "Large memory pages" mode in + Windows 10 because of some BUG with "Large Pages" in Windows 10. + Now 7-Zip doesn't use "Large Pages" on Windows 10 up to revision 1709 (16299). + + +18.03 beta 2018-03-04 +------------------------- +- Asm\x86\LzmaDecOpt.asm: new optimized LZMA decoder written in asm + for x64 with about 30% higher speed than main version of LZMA decoder written in C. +- The speed for single-thread LZMA/LZMA2 decoder written in C was increased by 3%. +- 7-Zip now can use multi-threading for 7z/LZMA2 decoding, + if there are multiple independent data chunks in LZMA2 stream. +- 7-Zip now can use multi-threading for xz decoding, + if there are multiple blocks in xz stream. + + 17.00 beta 2017-04-29 ------------------------- - NewHandler.h / NewHandler.cpp: diff --git a/README.md b/README.md index b4474e9a..6af0ac17 100644 --- a/README.md +++ b/README.md @@ -15,13 +15,13 @@ You can install it in two ways: ## Codec overview -1. [Zstandard] v1.3.2 is a real-time compression algorithm, providing high compression ratios. It offers a very wide range of compression / speed trade-off, while being backed by a very fast decoder. +1. [Zstandard] v1.3.6 is a real-time compression algorithm, providing high compression ratios. It offers a very wide range of compression / speed trade-off, while being backed by a very fast decoder. - Levels: 1..22 -2. [Brotli] v.1.0.1 is a generic-purpose lossless compression algorithm that compresses data using a combination of a modern variant of the LZ77 algorithm, Huffman coding and 2nd order context modeling, with a compression ratio comparable to the best currently available general-purpose compression methods. It is similar in speed with deflate but offers more dense compression. +2. [Brotli] v.1.0.6 is a generic-purpose lossless compression algorithm that compresses data using a combination of a modern variant of the LZ77 algorithm, Huffman coding and 2nd order context modeling, with a compression ratio comparable to the best currently available general-purpose compression methods. It is similar in speed with deflate but offers more dense compression. - Levels: 0..11 -3. [LZ4] v1.8.0 is lossless compression algorithm, providing compression speed at 400 MB/s per core (0.16 Bytes/cycle). It features an extremely fast decoder, with speed in multiple GB/s per core (0.71 Bytes/cycle). A high compression derivative, called LZ4_HC, is available, trading customizable CPU time for compression ratio. +3. [LZ4] v1.8.3 is lossless compression algorithm, providing compression speed at 400 MB/s per core (0.16 Bytes/cycle). It features an extremely fast decoder, with speed in multiple GB/s per core (0.71 Bytes/cycle). A high compression derivative, called LZ4_HC, is available, trading customizable CPU time for compression ratio. - Levels: 1..12 4. [LZ5] v1.5 is a modification of LZ4 which gives a better ratio at cost of slower compression and decompression. @@ -44,7 +44,7 @@ You can install it in two ways: The output should look like this: ``` -7-Zip 17.01 ZS v1.3.2 R1 (x64) : Copyright (c) 1999-2017 Igor Pavlov +7-Zip 17.01 ZS v1.3.6 R1 (x64) : Copyright (c) 1999-2017 Igor Pavlov Libs: @@ -247,13 +247,13 @@ You find this project useful, maybe you consider a donation ;-) ## Version Information - 7-Zip ZS Version 17.01 - - [Brotli] Version 1.0.1 + - [Brotli] Version 1.0.6 - [Lizard] Version 1.0 - - [LZ4] Version 1.8.0 + - [LZ4] Version 1.8.3 - [LZ5] Version 1.5 - - [Zstandard] Version 1.3.2 + - [Zstandard] Version 1.3.6 -/TR 2017-10-31 +/TR 2018-10-21 [7-Zip]:http://www.7-zip.org/ [lzip]:http://www.nongnu.org/lzip/