xevion/easy7zip
1
0
Fork 0
mirror of https://github.com/Xevion/easy7zip.git synced 2025-12-10 10:07:13 -06:00
Code Issues Packages Projects Releases Wiki Activity

4.34 beta

Browse Source
This commit is contained in:
Igor Pavlov
2006-03-01 00:00:00 +00:00
committed by Kornel Lesiński
parent 02516d3fce
commit 0f60a4933b
77 changed files with 2173 additions and 832 deletions
Download Patch File Download Diff File Expand all files Collapse all files

24
7zip/Compress/BZip2/BZip2.dsp
View File

@@ -43,7 +43,7 @@ RSC=rc.exe
# PROP Ignore_Export_Lib 1
# PROP Target_Dir ""
# ADD BASE CPP /nologo /MT /W3 /GX /O2 /D "WIN32" /D "NDEBUG" /D "_WINDOWS" /D "_MBCS" /D "_USRDLL" /D "BZIP2_EXPORTS" /YX /FD /c
# ADD CPP /nologo /Gz /MD /W3 /GX /O1 /I "..\..\..\\" /D "WIN32" /D "NDEBUG" /D "_WINDOWS" /D "_MBCS" /D "_USRDLL" /D "BZIP2_EXPORTS" /D "BZ_NO_STDIO" /Yu"StdAfx.h" /FD /c
# ADD CPP /nologo /Gz /MD /W3 /GX /O1 /I "..\..\..\\" /D "NDEBUG" /D "BZ_NO_STDIO" /D "WIN32" /D "_WINDOWS" /D "_MBCS" /D "_USRDLL" /D "BZIP2_EXPORTS" /D "COMPRESS_BZIP2_MT" /Yu"StdAfx.h" /FD /c
# ADD BASE MTL /nologo /D "NDEBUG" /mktyplib203 /win32
# ADD MTL /nologo /D "NDEBUG" /mktyplib203 /win32
# ADD BASE RSC /l 0x419 /d "NDEBUG"
@@ -70,7 +70,7 @@ LINK32=link.exe
# PROP Ignore_Export_Lib 1
# PROP Target_Dir ""
# ADD BASE CPP /nologo /MTd /W3 /Gm /GX /ZI /Od /D "WIN32" /D "_DEBUG" /D "_WINDOWS" /D "_MBCS" /D "_USRDLL" /D "BZIP2_EXPORTS" /YX /FD /GZ /c
# ADD CPP /nologo /Gz /MDd /W3 /Gm /GX /ZI /Od /I "..\..\..\\" /D "WIN32" /D "_DEBUG" /D "_WINDOWS" /D "_MBCS" /D "_USRDLL" /D "BZIP2_EXPORTS" /Yu"StdAfx.h" /FD /GZ /c
# ADD CPP /nologo /Gz /MDd /W3 /Gm /GX /ZI /Od /I "..\..\..\\" /D "_DEBUG" /D "WIN32" /D "_WINDOWS" /D "_MBCS" /D "_USRDLL" /D "BZIP2_EXPORTS" /D "COMPRESS_BZIP2_MT" /Yu"StdAfx.h" /FD /GZ /c
# ADD BASE MTL /nologo /D "_DEBUG" /mktyplib203 /win32
# ADD MTL /nologo /D "_DEBUG" /mktyplib203 /win32
# ADD BASE RSC /l 0x419 /d "_DEBUG"
@@ -213,6 +213,26 @@ SOURCE=..\BWT\BlockSort.h
SOURCE=..\BWT\Mtf8.h
# End Source File
# End Group
# Begin Group "Windows"
# PROP Default_Filter ""
# Begin Source File
SOURCE=..\..\..\Windows\Synchronization.cpp
# End Source File
# Begin Source File
SOURCE=..\..\..\Windows\Synchronization.h
# End Source File
# Begin Source File
SOURCE=..\..\..\Windows\Thread.h
# End Source File
# End Group
# Begin Source File
SOURCE=.\BZip2Const.h
# End Source File
# Begin Source File
SOURCE=.\BZip2CRC.cpp

326
7zip/Compress/BZip2/BZip2Decoder.cpp
View File

@@ -12,6 +12,8 @@
namespace NCompress {
namespace NBZip2 {
const UInt32 kNumThreadsMax = 4;
static const UInt32 kBufferSize = (1 << 17);
static Int16 kRandNums[512] = {
@@ -69,11 +71,6 @@ static Int16 kRandNums[512] = {
936, 638
};
CState::~CState()
{
::BigFree(tt);
}
bool CState::Alloc()
{
if (tt == 0)
@@ -81,6 +78,109 @@ bool CState::Alloc()
return (tt != 0);
}
void CState::Free()
{
::BigFree(tt);
tt = 0;
}
#ifdef COMPRESS_BZIP2_MT
void CState::FinishStream()
{
Decoder->StreamWasFinished = true;
StreamWasFinishedEvent.Set();
Decoder->CS.Leave();
Decoder->CanStartWaitingEvent.Lock();
WaitingWasStartedEvent.Set();
}
DWORD CState::ThreadFunc()
{
while (true)
{
Decoder->CS.Enter();
if (Decoder->CloseThreads)
{
Decoder->CS.Leave();
return 0;
}
if (Decoder->StreamWasFinished)
{
FinishStream();
continue;
}
HRESULT res = S_OK;
try
{
UInt32 blockIndex = Decoder->NextBlockIndex;
UInt32 nextBlockIndex = blockIndex + 1;
if (nextBlockIndex == Decoder->NumThreads)
nextBlockIndex = 0;
Decoder->NextBlockIndex = nextBlockIndex;
bool wasFinished;
UInt32 crc;
res = Decoder->ReadSignatures(wasFinished, crc);
if (res != S_OK)
{
Decoder->Result = res;
FinishStream();
continue;
}
if (wasFinished)
{
Decoder->Result = res;
FinishStream();
continue;
}
res = Decoder->ReadBlock(Decoder->BlockSizeMax, *this);
UInt64 packSize = Decoder->m_InStream.GetProcessedSize();
if (res != S_OK)
{
Decoder->Result = res;
FinishStream();
continue;
}
Decoder->CS.Leave();
DecodeBlock1();
Decoder->m_States[blockIndex].CanWriteEvent.Lock();
if (DecodeBlock2(Decoder->m_OutStream) != crc)
{
Decoder->Result = S_FALSE;
FinishStream();
continue;
}
if (Decoder->Progress)
{
UInt64 unpackSize = Decoder->m_OutStream.GetProcessedSize();
res = Decoder->Progress->SetRatioInfo(&packSize, &unpackSize);
}
Decoder->m_States[nextBlockIndex].CanWriteEvent.Set();
}
catch(const CInBufferException &e) { res = e.ErrorCode; }
catch(const COutBufferException &e) { res = e.ErrorCode; }
catch(...) { res = E_FAIL; }
if (res != S_OK)
{
Decoder->Result = res;
FinishStream();
continue;
}
}
}
static DWORD WINAPI MFThread(void *threadCoderInfo)
{
return ((CState *)threadCoderInfo)->ThreadFunc();
}
#endif
UInt32 CDecoder::ReadBits(int numBits) { return m_InStream.ReadBits(numBits); }
Byte CDecoder::ReadByte() {return (Byte)ReadBits(8); }
bool CDecoder::ReadBit() { return ReadBits(1) != 0; }
@@ -201,7 +301,7 @@ HRESULT CDecoder::ReadBlock(UInt32 blockSizeMax, CState &state)
groupSize = kGroupSize;
huffmanDecoder = &m_HuffmanDecoders[state.m_Selectors[groupIndex++]];
}
groupSize--; \
groupSize--;
UInt32 nextSym = huffmanDecoder->DecodeSymbol(&m_InStream);
@@ -241,7 +341,7 @@ HRESULT CDecoder::ReadBlock(UInt32 blockSizeMax, CState &state)
return S_OK;
}
HRESULT CState::DecodeBlock(COutBuffer &m_OutStream)
void CState::DecodeBlock1()
{
UInt32 *charCounters = this->CharCounters;
{
@@ -259,8 +359,11 @@ HRESULT CState::DecodeBlock(COutBuffer &m_OutStream)
do
tt[charCounters[tt[i] & 0xFF]++] |= (i << 8);
while(++i < blockSize);
}
// Decode
UInt32 CState::DecodeBlock2(COutBuffer &m_OutStream)
{
UInt32 blockSize = this->BlockSize;
CBZip2CRC crc;
@@ -310,11 +413,122 @@ HRESULT CState::DecodeBlock(COutBuffer &m_OutStream)
m_OutStream.WriteByte(b);
}
while(--blockSize != 0);
return (StoredCRC == crc.GetDigest()) ? S_OK : S_FALSE;
return crc.GetDigest();
}
#ifdef COMPRESS_BZIP2_MT
CDecoder::CDecoder():
m_States(0)
{
m_NumThreadsPrev = 0;
NumThreads = 1;
CS.Enter();
}
CDecoder::~CDecoder()
{
Free();
}
bool CDecoder::Create()
{
try
{
if (m_States != 0 && m_NumThreadsPrev == NumThreads)
return true;
Free();
MtMode = (NumThreads > 1);
m_NumThreadsPrev = NumThreads;
m_States = new CState[NumThreads];
if (m_States == 0)
return false;
#ifdef COMPRESS_BZIP2_MT
for (UInt32 t = 0; t < NumThreads; t++)
{
CState &ti = m_States[t];
ti.Decoder = this;
if (MtMode)
if (!ti.Thread.Create(MFThread, &ti))
{
NumThreads = t;
Free();
return false;
}
}
#endif
}
catch(...) { return false; }
return true;
}
void CDecoder::Free()
{
if (!m_States)
return;
CloseThreads = true;
CS.Leave();
for (UInt32 t = 0; t < NumThreads; t++)
{
CState &s = m_States[t];
if (MtMode)
s.Thread.Wait();
s.Free();
}
delete []m_States;
m_States = 0;
}
#endif
HRESULT CDecoder::ReadSignatures(bool &wasFinished, UInt32 &crc)
{
wasFinished = false;
Byte s[6];
for (int i = 0; i < 6; i++)
s[i] = ReadByte();
crc = ReadCRC();
if (s[0] == kFinSig0)
{
if (s[1] != kFinSig1 ||
s[2] != kFinSig2 ||
s[3] != kFinSig3 ||
s[4] != kFinSig4 ||
s[5] != kFinSig5)
return S_FALSE;
wasFinished = true;
return (crc == CombinedCRC.GetDigest()) ? S_OK : S_FALSE;
}
if (s[0] != kBlockSig0 ||
s[1] != kBlockSig1 ||
s[2] != kBlockSig2 ||
s[3] != kBlockSig3 ||
s[4] != kBlockSig4 ||
s[5] != kBlockSig5)
return S_FALSE;
CombinedCRC.Update(crc);
return S_OK;
}
HRESULT CDecoder::DecodeFile(bool &isBZ, ICompressProgressInfo *progress)
{
#ifdef COMPRESS_BZIP2_MT
Progress = progress;
if (!Create())
return E_FAIL;
for (UInt32 t = 0; t < NumThreads; t++)
{
CState &s = m_States[t];
if (!s.Alloc())
return E_OUTOFMEMORY;
s.StreamWasFinishedEvent.Reset();
s.WaitingWasStartedEvent.Reset();
s.CanWriteEvent.Reset();
}
#else
if (!m_States[0].Alloc())
return E_OUTOFMEMORY;
#endif
isBZ = false;
Byte s[6];
int i;
@@ -329,45 +543,53 @@ HRESULT CDecoder::DecodeFile(bool &isBZ, ICompressProgressInfo *progress)
isBZ = true;
UInt32 dicSize = (UInt32)(s[3] - kArSig3) * kBlockSizeStep;
if (!m_State.Alloc())
return E_OUTOFMEMORY;
CBZip2CombinedCRC computedCombinedCRC;
while (true)
CombinedCRC.Init();
#ifdef COMPRESS_BZIP2_MT
if (MtMode)
{
if (progress)
{
UInt64 packSize = m_InStream.GetProcessedSize();
UInt64 unpackSize = m_OutStream.GetProcessedSize();
RINOK(progress->SetRatioInfo(&packSize, &unpackSize));
}
for (i = 0; i < 6; i++)
s[i] = ReadByte();
UInt32 crc = ReadCRC();
if (s[0] == kFinSig0)
{
if (s[1] != kFinSig1 ||
s[2] != kFinSig2 ||
s[3] != kFinSig3 ||
s[4] != kFinSig4 ||
s[5] != kFinSig5)
return S_FALSE;
return (crc == computedCombinedCRC.GetDigest()) ? S_OK : S_FALSE;
}
if (s[0] != kBlockSig0 ||
s[1] != kBlockSig1 ||
s[2] != kBlockSig2 ||
s[3] != kBlockSig3 ||
s[4] != kBlockSig4 ||
s[5] != kBlockSig5)
return S_FALSE;
m_State.StoredCRC = crc;
computedCombinedCRC.Update(crc);
RINOK(ReadBlock(dicSize, m_State));
RINOK(m_State.DecodeBlock(m_OutStream));
NextBlockIndex = 0;
StreamWasFinished = false;
CloseThreads = false;
CanStartWaitingEvent.Reset();
m_States[0].CanWriteEvent.Set();
BlockSizeMax = dicSize;
Result = S_OK;
CS.Leave();
UInt32 t;
for (t = 0; t < NumThreads; t++)
m_States[t].StreamWasFinishedEvent.Lock();
CS.Enter();
CanStartWaitingEvent.Set();
for (t = 0; t < NumThreads; t++)
m_States[t].WaitingWasStartedEvent.Lock();
CanStartWaitingEvent.Reset();
RINOK(Result);
}
else
#endif
{
CState &state = m_States[0];
while (true)
{
if (progress)
{
UInt64 packSize = m_InStream.GetProcessedSize();
UInt64 unpackSize = m_OutStream.GetProcessedSize();
RINOK(progress->SetRatioInfo(&packSize, &unpackSize));
}
bool wasFinished;
UInt32 crc;
RINOK(ReadSignatures(wasFinished, crc));
if (wasFinished)
return S_OK;
RINOK(ReadBlock(dicSize, state));
state.DecodeBlock1();
if (state.DecodeBlock2(m_OutStream) != crc)
return S_FALSE;
}
}
return S_OK;
}
HRESULT CDecoder::CodeReal(ISequentialInStream *inStream,
@@ -408,4 +630,16 @@ STDMETHODIMP CDecoder::GetInStreamProcessedSize(UInt64 *value)
return S_OK;
}
#ifdef COMPRESS_BZIP2_MT
STDMETHODIMP CDecoder::SetNumberOfThreads(UInt32 numThreads)
{
NumThreads = numThreads;
if (NumThreads < 1)
NumThreads = 1;
if (NumThreads > kNumThreadsMax)
NumThreads = kNumThreadsMax;
return S_OK;
}
#endif
}}

88
7zip/Compress/BZip2/BZip2Decoder.h
View File

@@ -10,12 +10,20 @@
#include "../../Common/OutBuffer.h"
#include "../Huffman/HuffmanDecoder.h"
#include "BZip2Const.h"
#include "BZip2CRC.h"
#ifdef COMPRESS_BZIP2_MT
#include "../../../Windows/Thread.h"
#include "../../../Windows/Synchronization.h"
#endif
namespace NCompress {
namespace NBZip2 {
typedef NCompress::NHuffman::CDecoder<kMaxHuffmanLen, kMaxAlphaSize> CHuffmanDecoder;
class CDecoder;
struct CState
{
UInt32 *tt;
@@ -24,31 +32,55 @@ struct CState
UInt32 BlockSize;
UInt32 CharCounters[256];
Byte m_Selectors[kNumSelectorsMax];
UInt32 StoredCRC;
#ifdef COMPRESS_BZIP2_MT
CDecoder *Decoder;
NWindows::CThread Thread;
bool m_OptimizeNumTables;
NWindows::NSynchronization::CAutoResetEvent StreamWasFinishedEvent;
NWindows::NSynchronization::CAutoResetEvent WaitingWasStartedEvent;
// it's not member of this thread. We just need one event per thread
NWindows::NSynchronization::CAutoResetEvent CanWriteEvent;
Byte MtPad[1 << 8]; // It's pad for Multi-Threading. Must be >= Cache_Line_Size.
void FinishStream();
DWORD ThreadFunc();
#endif
CState(): tt(0) {}
~CState();
~CState() { Free(); }
bool Alloc();
HRESULT DecodeBlock(COutBuffer &m_OutStream);
void Free();
void DecodeBlock1();
UInt32 DecodeBlock2(COutBuffer &m_OutStream);
};
class CDecoder :
public ICompressCoder,
#ifdef COMPRESS_BZIP2_MT
public ICompressSetCoderMt,
#endif
public ICompressGetInStreamProcessedSize,
public CMyUnknownImp
{
NStream::NMSBF::CDecoder<CInBuffer> m_InStream;
public:
COutBuffer m_OutStream;
Byte MtPad[1 << 8]; // It's pad for Multi-Threading. Must be >= Cache_Line_Size.
NStream::NMSBF::CDecoder<CInBuffer> m_InStream;
private:
CHuffmanDecoder m_HuffmanDecoders[kNumTablesMax];
CState m_State;
UInt32 m_NumThreadsPrev;
UInt32 ReadBits(int numBits);
Byte ReadByte();
bool ReadBit();
UInt32 ReadCRC();
HRESULT ReadBlock(UInt32 blockSizeMax, CState &state);
HRESULT PrepareBlock(CState &state);
HRESULT DecodeFile(bool &isBZ, ICompressProgressInfo *progress);
HRESULT CodeReal(ISequentialInStream *inStream,
@@ -69,6 +101,37 @@ class CDecoder :
};
public:
CBZip2CombinedCRC CombinedCRC;
#ifdef COMPRESS_BZIP2_MT
ICompressProgressInfo *Progress;
CState *m_States;
NWindows::NSynchronization::CCriticalSection CS;
UInt32 NumThreads;
bool MtMode;
UInt32 NextBlockIndex;
bool CloseThreads;
bool StreamWasFinished;
NWindows::NSynchronization::CManualResetEvent CanStartWaitingEvent;
HRESULT Result;
UInt32 BlockSizeMax;
CDecoder();
~CDecoder();
bool Create();
void Free();
#else
CState m_States[1];
#endif
HRESULT ReadBlock(UInt32 blockSizeMax, CState &state);
HRESULT ReadSignatures(bool &wasFinished, UInt32 &crc);
HRESULT Flush() { return m_OutStream.Flush(); }
void ReleaseStreams()
{
@@ -76,13 +139,22 @@ public:
m_OutStream.ReleaseStream();
}
#ifdef COMPRESS_BZIP2_MT
MY_UNKNOWN_IMP2(ICompressSetCoderMt, ICompressGetInStreamProcessedSize)
#else
MY_UNKNOWN_IMP1(ICompressGetInStreamProcessedSize)
#endif
STDMETHOD(Code)(ISequentialInStream *inStream,
ISequentialOutStream *outStream, const UInt64 *inSize, const UInt64 *outSize,
ICompressProgressInfo *progress);
STDMETHOD(GetInStreamProcessedSize)(UInt64 *value);
#ifdef COMPRESS_BZIP2_MT
STDMETHOD(SetNumberOfThreads)(UInt32 numThreads);
#endif
};
}}

358
7zip/Compress/BZip2/BZip2Encoder.cpp
View File

@@ -2,9 +2,10 @@
#include "StdAfx.h"
#include "../../../Common/Alloc.h"
#include "BZip2Encoder.h"
#include "../../../Common/Alloc.h"
#include "../BWT/Mtf8.h"
#include "BZip2CRC.h"
@@ -14,19 +15,161 @@ namespace NBZip2 {
static const UInt32 kBufferSize = (1 << 17);
static const int kNumHuffPasses = 4;
CEncoder::CEncoder():
m_Block(0),
m_NeedHuffmanCreate(true),
m_NumPasses(1),
bool CThreadInfo::Create()
{
if (!m_BlockSorter.Create(kBlockSizeMax))
return false;
if (m_Block == 0)
{
m_Block = (Byte *)::BigAlloc(kBlockSizeMax * 5 + kBlockSizeMax / 10 + (20 << 10));
if (m_Block == 0)
return false;
m_MtfArray = m_Block + kBlockSizeMax;
m_TempArray = m_MtfArray + kBlockSizeMax * 2 + 2;
}
if (m_NeedHuffmanCreate)
{
for (int i = 0; i < kNumTablesMax; i++)
if (!m_HuffEncoders[i].Create(kMaxAlphaSize, 0, 0, kMaxHuffmanLen))
return false;
m_NeedHuffmanCreate = false;
}
return true;
}
void CThreadInfo::Free()
{
m_BlockSorter.Free();
::BigFree(m_Block);
m_Block = 0;
}
#ifdef COMPRESS_BZIP2_MT
void CThreadInfo::FinishStream()
{
Encoder->StreamWasFinished = true;
StreamWasFinishedEvent.Set();
Encoder->CS.Leave();
Encoder->CanStartWaitingEvent.Lock();
WaitingWasStartedEvent.Set();
}
DWORD CThreadInfo::ThreadFunc()
{
while (true)
{
Encoder->CS.Enter();
if (Encoder->CloseThreads)
{
Encoder->CS.Leave();
return 0;
}
if (Encoder->StreamWasFinished)
{
FinishStream();
continue;
}
HRESULT res = S_OK;
try
{
UInt32 blockSize = Encoder->ReadRleBlock(m_Block);
m_PackSize = Encoder->m_InStream.GetProcessedSize();
m_BlockIndex = Encoder->NextBlockIndex;
if (++Encoder->NextBlockIndex == Encoder->NumThreads)
Encoder->NextBlockIndex = 0;
if (blockSize == 0)
{
FinishStream();
continue;
}
Encoder->CS.Leave();
res = EncodeBlock3(blockSize);
}
catch(const CInBufferException &e) { res = e.ErrorCode; }
catch(const COutBufferException &e) { res = e.ErrorCode; }
catch(...) { res = E_FAIL; }
if (res != S_OK)
{
Encoder->Result = res;
FinishStream();
continue;
}
}
}
static DWORD WINAPI MFThread(void *threadCoderInfo)
{
return ((CThreadInfo *)threadCoderInfo)->ThreadFunc();
}
#endif
CEncoder::CEncoder():
NumPasses(1),
m_OptimizeNumTables(false),
m_BlockSizeMult(kBlockSizeMultMax)
{}
{
#ifdef COMPRESS_BZIP2_MT
ThreadsInfo = 0;
m_NumThreadsPrev = 0;
NumThreads = 1;
CS.Enter();
#endif
}
#ifdef COMPRESS_BZIP2_MT
CEncoder::~CEncoder()
{
::BigFree(m_Block);
Free();
}
bool CEncoder::Create()
{
try
{
if (ThreadsInfo != 0 && m_NumThreadsPrev == NumThreads)
return true;
Free();
MtMode = (NumThreads > 1);
m_NumThreadsPrev = NumThreads;
ThreadsInfo = new CThreadInfo[NumThreads];
if (ThreadsInfo == 0)
return false;
for (UInt32 t = 0; t < NumThreads; t++)
{
CThreadInfo &ti = ThreadsInfo[t];
ti.Encoder = this;
if (MtMode)
if (!ti.Thread.Create(MFThread, &ti))
{
NumThreads = t;
Free();
return false;
}
}
}
catch(...) { return false; }
return true;
}
void CEncoder::Free()
{
if (!ThreadsInfo)
return;
CloseThreads = true;
CS.Leave();
for (UInt32 t = 0; t < NumThreads; t++)
{
CThreadInfo &ti = ThreadsInfo[t];
if (MtMode)
ti.Thread.Wait();
ti.Free();
}
delete []ThreadsInfo;
ThreadsInfo = 0;
}
#endif
UInt32 CEncoder::ReadRleBlock(Byte *buffer)
{
UInt32 i = 0;
@@ -66,11 +209,11 @@ UInt32 CEncoder::ReadRleBlock(Byte *buffer)
return i;
}
void CEncoder::WriteBits2(UInt32 value, UInt32 numBits)
void CThreadInfo::WriteBits2(UInt32 value, UInt32 numBits)
{ m_OutStreamCurrent->WriteBits(value, numBits); }
void CEncoder::WriteByte2(Byte b) { WriteBits2(b , 8); }
void CEncoder::WriteBit2(bool v) { WriteBits2((v ? 1 : 0), 1); }
void CEncoder::WriteCRC2(UInt32 v)
void CThreadInfo::WriteByte2(Byte b) { WriteBits2(b , 8); }
void CThreadInfo::WriteBit2(bool v) { WriteBits2((v ? 1 : 0), 1); }
void CThreadInfo::WriteCRC2(UInt32 v)
{
for (int i = 0; i < 4; i++)
WriteByte2(((Byte)(v >> (24 - i * 8))));
@@ -88,7 +231,7 @@ void CEncoder::WriteCRC(UInt32 v)
// blockSize > 0
void CEncoder::EncodeBlock(Byte *block, UInt32 blockSize)
void CThreadInfo::EncodeBlock(Byte *block, UInt32 blockSize)
{
WriteBit2(false); // Randomised = false
@@ -404,7 +547,7 @@ void CEncoder::EncodeBlock(Byte *block, UInt32 blockSize)
}
// blockSize > 0
UInt32 CEncoder::EncodeBlockWithHeaders(Byte *block, UInt32 blockSize)
UInt32 CThreadInfo::EncodeBlockWithHeaders(Byte *block, UInt32 blockSize)
{
WriteByte2(kBlockSig0);
WriteByte2(kBlockSig1);
@@ -443,11 +586,10 @@ UInt32 CEncoder::EncodeBlockWithHeaders(Byte *block, UInt32 blockSize)
return crcRes;
}
void CEncoder::EncodeBlock2(CBZip2CombinedCRC &combinedCRC,
Byte *block, UInt32 blockSize, UInt32 numPasses)
void CThreadInfo::EncodeBlock2(Byte *block, UInt32 blockSize, UInt32 numPasses)
{
UInt32 numCrcs = m_NumCrcs;
bool needCompare = false;
CBZip2CombinedCRC specCombinedCRC = combinedCRC;
UInt32 startBytePos = m_OutStreamCurrent->GetBytePos();
UInt32 startPos = m_OutStreamCurrent->GetPos();
@@ -462,9 +604,8 @@ void CEncoder::EncodeBlock2(CBZip2CombinedCRC &combinedCRC,
blockSize0 < blockSize; blockSize0++);
if (blockSize0 < blockSize)
{
EncodeBlock2(specCombinedCRC, block, blockSize0, numPasses - 1);
EncodeBlock2(specCombinedCRC, block + blockSize0, blockSize - blockSize0,
numPasses - 1);
EncodeBlock2(block, blockSize0, numPasses - 1);
EncodeBlock2(block + blockSize0, blockSize - blockSize0, numPasses - 1);
endPos = m_OutStreamCurrent->GetPos();
endCurByte = m_OutStreamCurrent->GetCurByte();
if ((endPos & 7) > 0)
@@ -479,8 +620,6 @@ void CEncoder::EncodeBlock2(CBZip2CombinedCRC &combinedCRC,
UInt32 crcVal = EncodeBlockWithHeaders(block, blockSize);
UInt32 endPos2 = m_OutStreamCurrent->GetPos();
combinedCRC.Update(crcVal);
if (needCompare)
{
UInt32 size2 = endPos2 - startPos2;
@@ -491,59 +630,102 @@ void CEncoder::EncodeBlock2(CBZip2CombinedCRC &combinedCRC,
for (UInt32 i = 0; i < numBytes; i++)
buffer[startBytePos + i] = buffer[startBytePos2 + i];
m_OutStreamCurrent->SetPos(startPos + endPos2 - startPos2);
m_NumCrcs = numCrcs;
m_CRCs[m_NumCrcs++] = crcVal;
}
else
{
m_OutStreamCurrent->SetPos(endPos);
m_OutStreamCurrent->SetCurState((endPos & 7), endCurByte);
combinedCRC = specCombinedCRC;
}
}
else
{
m_NumCrcs = numCrcs;
m_CRCs[m_NumCrcs++] = crcVal;
}
}
void CEncoder::EncodeBlock3(CBZip2CombinedCRC &combinedCRC, UInt32 blockSize)
HRESULT CThreadInfo::EncodeBlock3(UInt32 blockSize)
{
CMsbfEncoderTemp outStreamTemp;
outStreamTemp.SetStream(m_TempArray);
outStreamTemp.Init();
m_OutStreamCurrent = &outStreamTemp;
EncodeBlock2(combinedCRC, m_Block, blockSize, m_NumPasses);
m_NumCrcs = 0;
UInt32 size = outStreamTemp.GetPos();
UInt32 bytesSize = (size / 8);
for (UInt32 i = 0; i < bytesSize; i++)
m_OutStream.WriteBits(m_TempArray[i], 8);
WriteBits(outStreamTemp.GetCurByte(), (size & 7));
EncodeBlock2(m_Block, blockSize, Encoder->NumPasses);
#ifdef COMPRESS_BZIP2_MT
if (Encoder->MtMode)
Encoder->ThreadsInfo[m_BlockIndex].CanWriteEvent.Lock();
#endif
for (UInt32 i = 0; i < m_NumCrcs; i++)
Encoder->CombinedCRC.Update(m_CRCs[i]);
Encoder->WriteBytes(m_TempArray, outStreamTemp.GetPos(), outStreamTemp.GetCurByte());
HRESULT res = S_OK;
#ifdef COMPRESS_BZIP2_MT
if (Encoder->MtMode)
{
UInt32 blockIndex = m_BlockIndex + 1;
if (blockIndex == Encoder->NumThreads)
blockIndex = 0;
if (Encoder->Progress)
{
UInt64 unpackSize = Encoder->m_OutStream.GetProcessedSize();
res = Encoder->Progress->SetRatioInfo(&m_PackSize, &unpackSize);
}
Encoder->ThreadsInfo[blockIndex].CanWriteEvent.Set();
}
#endif
return res;
}
void CEncoder::WriteBytes(const Byte *data, UInt32 sizeInBits, Byte lastByte)
{
UInt32 bytesSize = (sizeInBits / 8);
for (UInt32 i = 0; i < bytesSize; i++)
m_OutStream.WriteBits(data[i], 8);
WriteBits(lastByte, (sizeInBits & 7));
}
HRESULT CEncoder::CodeReal(ISequentialInStream *inStream,
ISequentialOutStream *outStream, const UInt64 *inSize, const UInt64 *outSize,
ICompressProgressInfo *progress)
{
if (!m_BlockSorter.Create(kBlockSizeMax))
return E_OUTOFMEMORY;
if (m_Block == 0)
#ifdef COMPRESS_BZIP2_MT
Progress = progress;
if (!Create())
return E_FAIL;
for (UInt32 t = 0; t < NumThreads; t++)
#endif
{
m_Block = (Byte *)BigAlloc(kBlockSizeMax * 5 + kBlockSizeMax / 10 + (20 << 10));
if (m_Block == 0)
#ifdef COMPRESS_BZIP2_MT
CThreadInfo &ti = ThreadsInfo[t];
ti.StreamWasFinishedEvent.Reset();
ti.WaitingWasStartedEvent.Reset();
ti.CanWriteEvent.Reset();
#else
CThreadInfo &ti = ThreadsInfo;
ti.Encoder = this;
#endif
ti.m_OptimizeNumTables = m_OptimizeNumTables;
if (!ti.Create())
return E_OUTOFMEMORY;
m_MtfArray = m_Block + kBlockSizeMax;
m_TempArray = m_MtfArray + kBlockSizeMax * 2 + 2;
}
if (!m_InStream.Create(kBufferSize))
return E_OUTOFMEMORY;
if (!m_OutStream.Create(kBufferSize))
return E_OUTOFMEMORY;
if (m_NeedHuffmanCreate)
{
for (int i = 0; i < kNumTablesMax; i++)
if (!m_HuffEncoders[i].Create(kMaxAlphaSize, 0, 0, kMaxHuffmanLen))
return E_OUTOFMEMORY;
m_NeedHuffmanCreate = false;
}
m_InStream.SetStream(inStream);
m_InStream.Init();
@@ -553,24 +735,57 @@ HRESULT CEncoder::CodeReal(ISequentialInStream *inStream,
CFlusher flusher(this);
CBZip2CombinedCRC combinedCRC;
CombinedCRC.Init();
#ifdef COMPRESS_BZIP2_MT
NextBlockIndex = 0;
StreamWasFinished = false;
CloseThreads = false;
CanStartWaitingEvent.Reset();
#endif
WriteByte(kArSig0);
WriteByte(kArSig1);
WriteByte(kArSig2);
WriteByte((Byte)(kArSig3 + m_BlockSizeMult));
while (true)
#ifdef COMPRESS_BZIP2_MT
if (MtMode)
{
UInt32 blockSize = ReadRleBlock(m_Block);
if (blockSize == 0)
break;
EncodeBlock3(combinedCRC, blockSize);
if (progress)
ThreadsInfo[0].CanWriteEvent.Set();
Result = S_OK;
CS.Leave();
UInt32 t;
for (t = 0; t < NumThreads; t++)
ThreadsInfo[t].StreamWasFinishedEvent.Lock();
CS.Enter();
CanStartWaitingEvent.Set();
for (t = 0; t < NumThreads; t++)
ThreadsInfo[t].WaitingWasStartedEvent.Lock();
CanStartWaitingEvent.Reset();
RINOK(Result);
}
else
#endif
{
while (true)
{
UInt64 packSize = m_InStream.GetProcessedSize();
UInt64 unpackSize = m_OutStream.GetProcessedSize();
RINOK(progress->SetRatioInfo(&packSize, &unpackSize));
CThreadInfo &ti =
#ifdef COMPRESS_BZIP2_MT
ThreadsInfo[0];
#else
ThreadsInfo;
#endif
UInt32 blockSize = ReadRleBlock(ti.m_Block);
if (blockSize == 0)
break;
RINOK(ti.EncodeBlock3(blockSize));
if (progress)
{
UInt64 packSize = m_InStream.GetProcessedSize();
UInt64 unpackSize = m_OutStream.GetProcessedSize();
RINOK(progress->SetRatioInfo(&packSize, &unpackSize));
}
}
}
WriteByte(kFinSig0);
@@ -580,7 +795,7 @@ HRESULT CEncoder::CodeReal(ISequentialInStream *inStream,
WriteByte(kFinSig4);
WriteByte(kFinSig5);
WriteCRC(combinedCRC.GetDigest());
WriteCRC(CombinedCRC.GetDigest());
return S_OK;
}
@@ -589,6 +804,8 @@ STDMETHODIMP CEncoder::Code(ISequentialInStream *inStream,
ICompressProgressInfo *progress)
{
try { return CodeReal(inStream, outStream, inSize, outSize, progress); }
catch(const CInBufferException &e) { return e.ErrorCode; }
catch(const COutBufferException &e) { return e.ErrorCode; }
catch(...) { return S_FALSE; }
}
@@ -605,10 +822,12 @@ HRESULT CEncoder::SetCoderProperties(const PROPID *propIDs,
if (property.vt != VT_UI4)
return E_INVALIDARG;
UInt32 numPasses = property.ulVal;
if(numPasses == 0 || numPasses > 10)
return E_INVALIDARG;
m_NumPasses = numPasses;
m_OptimizeNumTables = (m_NumPasses > 1);
if (numPasses == 0)
numPasses = 1;
if (numPasses > kNumPassesMax)
numPasses = kNumPassesMax;
NumPasses = numPasses;
m_OptimizeNumTables = (NumPasses > 1);
break;
}
case NCoderPropID::kDictionarySize:
@@ -623,6 +842,17 @@ HRESULT CEncoder::SetCoderProperties(const PROPID *propIDs,
m_BlockSizeMult = dictionary;
break;
}
case NCoderPropID::kNumThreads:
{
#ifdef COMPRESS_BZIP2_MT
if (property.vt != VT_UI4)
return E_INVALIDARG;
NumThreads = property.ulVal;
if (NumThreads < 1)
NumThreads = 1;
#endif
break;
}
default:
return E_INVALIDARG;
}
@@ -630,4 +860,14 @@ HRESULT CEncoder::SetCoderProperties(const PROPID *propIDs,
return S_OK;
}
#ifdef COMPRESS_BZIP2_MT
STDMETHODIMP CEncoder::SetNumberOfThreads(UInt32 numThreads)
{
NumThreads = numThreads;
if (NumThreads < 1)
NumThreads = 1;
return S_OK;
}
#endif
}}

133
7zip/Compress/BZip2/BZip2Encoder.h
View File

@@ -12,7 +12,12 @@
#include "../BWT/BlockSort.h"
#include "BZip2Const.h"
#include "BZip2CRC.h"
#ifdef COMPRESS_BZIP2_MT
#include "../../../Windows/Thread.h"
#include "../../../Windows/Synchronization.h"
#endif
namespace NCompress {
namespace NBZip2 {
@@ -76,49 +81,123 @@ public:
}
};
class CEncoder :
public ICompressCoder,
public ICompressSetCoderProperties,
public CMyUnknownImp
class CEncoder;
const int kNumPassesMax = 10;
class CThreadInfo
{
public:
Byte *m_Block;
CInBuffer m_InStream;
NStream::NMSBF::CEncoder<COutBuffer> m_OutStream;
CMsbfEncoderTemp *m_OutStreamCurrent;
CBlockSorter m_BlockSorter;
bool m_NeedHuffmanCreate;
NCompression::NHuffman::CEncoder m_HuffEncoders[kNumTablesMax];
private:
Byte *m_MtfArray;
Byte *m_TempArray;
CBlockSorter m_BlockSorter;
CMsbfEncoderTemp *m_OutStreamCurrent;
NCompression::NHuffman::CEncoder m_HuffEncoders[kNumTablesMax];
Byte m_Selectors[kNumSelectorsMax];
UInt32 m_BlockSizeMult;
UInt32 m_NumPasses;
bool m_OptimizeNumTables;
bool m_NeedHuffmanCreate;
UInt32 ReadRleBlock(Byte *buffer);
UInt32 m_CRCs[1 << kNumPassesMax];
UInt32 m_NumCrcs;
int m_BlockIndex;
void FinishStream();
void WriteBits2(UInt32 value, UInt32 numBits);
void WriteByte2(Byte b);
void WriteBit2(bool v);
void WriteCRC2(UInt32 v);
void EncodeBlock(Byte *block, UInt32 blockSize);
UInt32 EncodeBlockWithHeaders(Byte *block, UInt32 blockSize);
void EncodeBlock2(Byte *block, UInt32 blockSize, UInt32 numPasses);
public:
bool m_OptimizeNumTables;
CEncoder *Encoder;
#ifdef COMPRESS_BZIP2_MT
NWindows::CThread Thread;
NWindows::NSynchronization::CAutoResetEvent StreamWasFinishedEvent;
NWindows::NSynchronization::CAutoResetEvent WaitingWasStartedEvent;
// it's not member of this thread. We just need one event per thread
NWindows::NSynchronization::CAutoResetEvent CanWriteEvent;
UInt64 m_PackSize;
Byte MtPad[1 << 8]; // It's pad for Multi-Threading. Must be >= Cache_Line_Size.
#endif
CThreadInfo(): m_Block(0), m_NeedHuffmanCreate(true) {}
~CThreadInfo() { Free(); }
bool Create();
void Free();
HRESULT EncodeBlock3(UInt32 blockSize);
DWORD ThreadFunc();
};
class CEncoder :
public ICompressCoder,
public ICompressSetCoderProperties,
#ifdef COMPRESS_BZIP2_MT
public ICompressSetCoderMt,
#endif
public CMyUnknownImp
{
UInt32 m_BlockSizeMult;
bool m_OptimizeNumTables;
UInt32 m_NumPassesPrev;
UInt32 m_NumThreadsPrev;
public:
CInBuffer m_InStream;
Byte MtPad[1 << 8]; // It's pad for Multi-Threading. Must be >= Cache_Line_Size.
NStream::NMSBF::CEncoder<COutBuffer> m_OutStream;
UInt32 NumPasses;
CBZip2CombinedCRC CombinedCRC;
#ifdef COMPRESS_BZIP2_MT
CThreadInfo *ThreadsInfo;
NWindows::NSynchronization::CCriticalSection CS;
UInt32 NumThreads;
bool MtMode;
UInt32 NextBlockIndex;
bool CloseThreads;
bool StreamWasFinished;
NWindows::NSynchronization::CManualResetEvent CanStartWaitingEvent;
HRESULT Result;
ICompressProgressInfo *Progress;
#else
CThreadInfo ThreadsInfo;
#endif
UInt32 ReadRleBlock(Byte *buffer);
void WriteBytes(const Byte *data, UInt32 sizeInBits, Byte lastByte);
void WriteBits(UInt32 value, UInt32 numBits);
void WriteByte(Byte b);
void WriteBit(bool v);
void WriteCRC(UInt32 v);
void EncodeBlock(Byte *block, UInt32 blockSize);
UInt32 EncodeBlockWithHeaders(Byte *block, UInt32 blockSize);
void EncodeBlock2(CBZip2CombinedCRC &combinedCRC, Byte *block, UInt32 blockSize, UInt32 numPasses);
void EncodeBlock3(CBZip2CombinedCRC &combinedCRC, UInt32 blockSize);
#ifdef COMPRESS_BZIP2_MT
bool Create();
void Free();
#endif
public:
CEncoder();
#ifdef COMPRESS_BZIP2_MT
~CEncoder();
#endif
HRESULT Flush() { return m_OutStream.Flush(); }
@@ -142,7 +221,11 @@ public:
}
};
MY_UNKNOWN_IMP1(ICompressSetCoderProperties)
#ifdef COMPRESS_BZIP2_MT
MY_UNKNOWN_IMP2(ICompressSetCoderMt, ICompressSetCoderProperties)
#else
MY_UNKNOWN_IMP1(ICompressGetInStreamProcessedSize)
#endif
HRESULT CodeReal(ISequentialInStream *inStream,
ISequentialOutStream *outStream, const UInt64 *inSize, const UInt64 *outSize,
@@ -153,6 +236,10 @@ public:
ICompressProgressInfo *progress);
STDMETHOD(SetCoderProperties)(const PROPID *propIDs,
const PROPVARIANT *properties, UInt32 numProperties);
#ifdef COMPRESS_BZIP2_MT
STDMETHOD(SetNumberOfThreads)(UInt32 numThreads);
#endif
};
}}

8
7zip/Compress/BZip2/makefile
View File

@@ -1,6 +1,6 @@
PROG = BZip2.dll
DEF_FILE = ../Codec.def
CFLAGS = $(CFLAGS) -I ../../../
CFLAGS = $(CFLAGS) -I ../../../ -DCOMPRESS_BZIP2_MT
LIBS = $(LIBS) oleaut32.lib
BZIP2_OBJS = \
@@ -14,6 +14,9 @@ BZIP2_OPT_OBJS = \
COMMON_OBJS = \
$O\Alloc.obj \
WIN_OBJS = \
$O\Synchronization.obj
7ZIP_COMMON_OBJS = \
$O\InBuffer.obj \
$O\OutBuffer.obj \
@@ -23,6 +26,7 @@ OBJS = \
$(BZIP2_OBJS) \
$(BZIP2_OPT_OBJS) \
$(COMMON_OBJS) \
$(WIN_OBJS) \
$(7ZIP_COMMON_OBJS) \
$O\BlockSort.obj \
$O\HuffmanEncoder.obj \
@@ -36,6 +40,8 @@ $(BZIP2_OPT_OBJS): $(*B).cpp
$(COMPL_O2)
$(COMMON_OBJS): ../../../Common/$(*B).cpp
$(COMPL)
$(WIN_OBJS): ../../../Windows/$(*B).cpp
$(COMPL)
$(7ZIP_COMMON_OBJS): ../../Common/$(*B).cpp
$(COMPL)
$O\BlockSort.obj: ../BWT/$(*B).cpp