4.33 beta

This commit is contained in:
Igor Pavlov
2006-02-05 00:00:00 +00:00
committed by Kornel Lesiński
parent e8d0636d7a
commit 02516d3fce
80 changed files with 2185 additions and 3957 deletions

View File

@@ -6,87 +6,96 @@
#include "MT.h"
class CMatchFinderCallback:
public IMatchFinderCallback,
public CMyUnknownImp
{
MY_UNKNOWN_IMP
static const UInt32 kBlockSize = (1 << 14);
STDMETHOD(BeforeChangingBufferPos)();
STDMETHOD(AfterChangingBufferPos)();
public:
CMatchFinderMT *m_MatchFinderMT;
const Byte *m_BufferPosBefore;
};
STDMETHODIMP CMatchFinderCallback::BeforeChangingBufferPos()
static DWORD WINAPI MFThread(void *threadCoderInfo)
{
m_MatchFinderMT->m_AskChangeBufferPos.Set();
m_MatchFinderMT->m_CanChangeBufferPos.Lock();
m_BufferPosBefore = m_MatchFinderMT->m_MatchFinder->GetPointerToCurrentPos();
return S_OK;
return ((CMatchFinderMT *)threadCoderInfo)->ThreadFunc();
}
STDMETHODIMP CMatchFinderCallback::AfterChangingBufferPos()
CMatchFinderMT::CMatchFinderMT():
m_Buffer(0),
m_NeedStart(true)
{
m_MatchFinderMT->m_DataCurrentPos +=
m_MatchFinderMT->m_MatchFinder->GetPointerToCurrentPos() - m_BufferPosBefore;
m_MatchFinderMT->m_BufferPosWasChanged.Set();
return S_OK;
m_BlockIndex = kNumMTBlocks - 1;
m_CS[m_BlockIndex].Enter();
if (!m_Thread.Create(MFThread, this))
throw 271826;
}
HRESULT CMatchFinderMT::SetMatchFinder(IMatchFinder *matchFinder,
UInt32 multiThreadMult)
CMatchFinderMT::~CMatchFinderMT()
{
_multiThreadMult = multiThreadMult;
m_MatchFinder = matchFinder;
CMyComPtr<IMatchFinderSetCallback> matchFinderSetCallback;
if (m_MatchFinder.QueryInterface(IID_IMatchFinderSetCallback,
&matchFinderSetCallback) == S_OK)
{
CMatchFinderCallback *matchFinderCallbackSpec =
new CMatchFinderCallback;
CMyComPtr<IMatchFinderCallback> matchFinderCallback = matchFinderCallbackSpec;
matchFinderCallbackSpec->m_MatchFinderMT = this;
matchFinderSetCallback->SetCallback(matchFinderCallback);
return S_OK;
}
else
return E_FAIL;
m_Exit = true;
m_CS[m_BlockIndex].Leave();
m_CanChangeBufferPos.Set();
if (m_NeedStart)
m_MtCanStart.Set();
m_Thread.Wait();
FreeMem();
}
void CMatchFinderMT::FreeMem()
{
::MyFree(m_Buffer);
m_Buffer = 0;
}
STDMETHODIMP CMatchFinderMT::Init(ISequentialInStream *s)
STDMETHODIMP CMatchFinderMT::Create(UInt32 sizeHistory, UInt32 keepAddBufferBefore,
UInt32 matchMaxLen, UInt32 keepAddBufferAfter)
{
m_AskChangeBufferPos.Reset();
m_CanChangeBufferPos.Reset();
m_BufferPosWasChanged.Reset();
m_StopWriting.Reset();
m_WritingWasStopped.Reset();
m_NeedStart = true;
m_CurrentPos = 0;
m_CurrentLimitPos = 0;
FreeMem();
m_MatchMaxLen = matchMaxLen;
if (kBlockSize <= matchMaxLen * 4)
return E_INVALIDARG;
UInt32 bufferSize = kBlockSize * kNumMTBlocks;
m_Buffer = (UInt32 *)::MyAlloc(bufferSize * sizeof(UInt32));
if (m_Buffer == 0)
return E_OUTOFMEMORY;
keepAddBufferBefore += bufferSize;
keepAddBufferAfter += (kBlockSize + 1);
return m_MatchFinder->Create(sizeHistory, keepAddBufferBefore, matchMaxLen, keepAddBufferAfter);
}
HRESULT result = m_MatchFinder->Init(s);
// UInt32 blockSizeMult = 800
HRESULT CMatchFinderMT::SetMatchFinder(IMatchFinder *matchFinder)
{
m_MatchFinder = matchFinder;
return S_OK;
}
STDMETHODIMP CMatchFinderMT::SetStream(ISequentialInStream *s)
{
return m_MatchFinder->SetStream(s);
}
// Call it after ReleaseStream / SetStream
STDMETHODIMP CMatchFinderMT::Init()
{
m_NeedStart = true;
m_Pos = 0;
m_PosLimit = 0;
HRESULT result = m_MatchFinder->Init();
if (result == S_OK)
m_DataCurrentPos = m_MatchFinder->GetPointerToCurrentPos();
m_NumAvailableBytes = m_MatchFinder->GetNumAvailableBytes();
return result;
}
// ReleaseStream is required to finish multithreading
STDMETHODIMP_(void) CMatchFinderMT::ReleaseStream()
{
m_StopWriting.Set();
m_WritingWasStopped.Lock();
m_MatchFinder->ReleaseStream();
}
STDMETHODIMP CMatchFinderMT::MovePos()
{
if (m_Result != S_OK)
return m_Result;
m_NumAvailableBytesCurrent--;
m_DataCurrentPos++;
return S_OK;
m_StopWriting = true;
m_CS[m_BlockIndex].Leave();
if (!m_NeedStart)
{
m_CanChangeBufferPos.Set();
m_MtWasStopped.Lock();
m_NeedStart = true;
}
m_MatchFinder->ReleaseStream();
m_BlockIndex = kNumMTBlocks - 1;
m_CS[m_BlockIndex].Enter();
}
STDMETHODIMP_(Byte) CMatchFinderMT::GetIndexByte(Int32 index)
@@ -94,11 +103,10 @@ STDMETHODIMP_(Byte) CMatchFinderMT::GetIndexByte(Int32 index)
return m_DataCurrentPos[index];
}
STDMETHODIMP_(UInt32) CMatchFinderMT::GetMatchLen(Int32 index,
UInt32 distance, UInt32 limit)
STDMETHODIMP_(UInt32) CMatchFinderMT::GetMatchLen(Int32 index, UInt32 distance, UInt32 limit)
{
if (int(index + limit) > m_NumAvailableBytesCurrent)
limit = m_NumAvailableBytesCurrent - (index);
if ((Int32)(index + limit) > m_NumAvailableBytes)
limit = m_NumAvailableBytes - (index);
distance++;
const Byte *pby = m_DataCurrentPos + index;
UInt32 i;
@@ -111,202 +119,177 @@ STDMETHODIMP_(const Byte *) CMatchFinderMT::GetPointerToCurrentPos()
return m_DataCurrentPos;
}
STDMETHODIMP_(UInt32) CMatchFinderMT::GetNumAvailableBytes()
{
if (m_NeedStart)
return m_MatchFinder->GetNumAvailableBytes();
else
return m_NumAvailableBytesCurrent;
return m_NumAvailableBytes;
}
void CMatchFinderMT::FreeMem()
void CMatchFinderMT::GetNextBlock()
{
MyFree(m_Buffer);
MyFree(m_DummyBuffer);
if (m_NeedStart)
{
m_NeedStart = false;
for (UInt32 i = 0; i < kNumMTBlocks; i++)
m_StopReading[i] = false;
m_StopWriting = false;
m_Exit = false;
m_MtWasStarted.Reset();
m_MtWasStopped.Reset();
m_CanChangeBufferPos.Reset();
m_BufferPosWasChanged.Reset();
m_MtCanStart.Set();
m_MtWasStarted.Lock();
m_Result = S_OK;
}
while (true)
{
UInt32 nextIndex = (m_BlockIndex == kNumMTBlocks - 1) ? 0 : m_BlockIndex + 1;
m_CS[nextIndex].Enter();
if (!m_StopReading[nextIndex])
{
m_CS[m_BlockIndex].Leave();
m_BlockIndex = nextIndex;
break;
}
m_StopReading[nextIndex] = false;
m_CS[nextIndex].Leave();
m_CanChangeBufferPos.Set();
m_BufferPosWasChanged.Lock();
m_CS[nextIndex].Enter();
m_CS[m_BlockIndex].Leave();
m_BlockIndex = nextIndex;
}
m_Pos = m_BlockIndex * kBlockSize;
m_PosLimit = m_Buffer[m_Pos++];
m_NumAvailableBytes = m_Buffer[m_Pos++];
m_Result = m_Results[m_BlockIndex];
}
STDMETHODIMP CMatchFinderMT::Create(UInt32 sizeHistory,
UInt32 keepAddBufferBefore, UInt32 matchMaxLen,
UInt32 keepAddBufferAfter)
STDMETHODIMP CMatchFinderMT::GetMatches(UInt32 *distances)
{
FreeMem();
m_MatchMaxLen = matchMaxLen;
if (m_Pos == m_PosLimit)
GetNextBlock();
m_BlockSize = (matchMaxLen + 1) * _multiThreadMult;
UInt32 bufferSize = m_BlockSize * kNumMTBlocks;
m_DummyBuffer = (UInt32 *)MyAlloc((matchMaxLen + 1) * sizeof(UInt32));
if (m_DummyBuffer == 0)
return E_OUTOFMEMORY;
m_Buffer = (UInt32 *)MyAlloc(bufferSize * sizeof(UInt32));
if (m_Buffer == 0)
return E_OUTOFMEMORY;
for (int i = 0; i < kNumMTBlocks; i++)
m_Buffers[i] = &m_Buffer[i * m_BlockSize];
if (m_Result != S_OK)
return m_Result;
m_NumAvailableBytes--;
m_DataCurrentPos++;
m_NeedStart = true;
m_CurrentPos = 0;
m_CurrentLimitPos = 0;
keepAddBufferBefore += bufferSize;
return m_MatchFinder->Create(sizeHistory, keepAddBufferBefore, matchMaxLen,
keepAddBufferAfter);
const UInt32 *buffer = m_Buffer + m_Pos;
UInt32 len = *buffer++;
*distances++ = len;
m_Pos += 1 + len;
for (UInt32 i = 0; i != len; i += 2)
{
distances[i] = buffer[i];
distances[i + 1] = buffer[i + 1];
}
return S_OK;
}
static DWORD WINAPI MFThread(void *threadCoderInfo)
STDMETHODIMP CMatchFinderMT::Skip(UInt32 num)
{
do
{
if (m_Pos == m_PosLimit)
GetNextBlock();
if (m_Result != S_OK)
return m_Result;
m_NumAvailableBytes--;
m_DataCurrentPos++;
UInt32 len = m_Buffer[m_Pos++];
m_Pos += len;
}
while(--num != 0);
return S_OK;
}
STDMETHODIMP_(Int32) CMatchFinderMT::NeedChangeBufferPos(UInt32 numCheckBytes)
{
CMatchFinderMT &mt = *(CMatchFinderMT *)threadCoderInfo;
return mt.ThreadFunc();
throw 1;
}
STDMETHODIMP_(void) CMatchFinderMT::ChangeBufferPos()
{
throw 1;
}
DWORD CMatchFinderMT::ThreadFunc()
{
bool errorMode = false;
while (true)
while(true)
{
HANDLE events[3] = { m_ExitEvent, m_StopWriting, m_CanWriteEvents[m_WriteBufferIndex] } ;
DWORD waitResult = ::WaitForMultipleObjects((errorMode ? 2: 3), events, FALSE, INFINITE);
if (waitResult == WAIT_OBJECT_0 + 0)
return 0;
if (waitResult == WAIT_OBJECT_0 + 1)
{
m_WriteBufferIndex = 0;
for (int i = 0; i < kNumMTBlocks; i++)
m_CanWriteEvents[i].Reset();
m_WritingWasStopped.Set();
errorMode = false;
continue;
}
if (errorMode)
{
// this case means bug_in_program. So just exit;
return 1;
}
m_Results[m_WriteBufferIndex] = S_OK;
UInt32 *buffer = m_Buffers[m_WriteBufferIndex];
UInt32 curPos = 0;
UInt32 numBytes = 0;
UInt32 limit = m_BlockSize - m_MatchMaxLen;
IMatchFinder *mf = m_MatchFinder;
do
{
if (mf->GetNumAvailableBytes() == 0)
break;
UInt32 len = mf->GetLongestMatch(buffer + curPos);
/*
if (len == 1)
len = 0;
*/
buffer[curPos] = len;
curPos += len + 1;
numBytes++;
HRESULT result = mf->MovePos();
if (result != S_OK)
{
m_Results[m_WriteBufferIndex] = result;
errorMode = true;
break;
}
}
while (curPos < limit);
m_LimitPos[m_WriteBufferIndex] = curPos;
if (errorMode)
m_NumAvailableBytes[m_WriteBufferIndex] = numBytes;
else
m_NumAvailableBytes[m_WriteBufferIndex] = numBytes +
mf->GetNumAvailableBytes();
m_CanReadEvents[m_WriteBufferIndex].Set();
if (++m_WriteBufferIndex == kNumMTBlocks)
m_WriteBufferIndex = 0;
}
}
CMatchFinderMT::CMatchFinderMT():
m_Buffer(0),
m_DummyBuffer(0),
_multiThreadMult(100)
{
for (int i = 0; i < kNumMTBlocks; i++)
{
m_CanReadEvents[i].Reset();
m_CanWriteEvents[i].Reset();
}
m_ReadBufferIndex = 0;
m_WriteBufferIndex = 0;
m_ExitEvent.Reset();
if (!m_Thread.Create(MFThread, this))
throw 271826;
}
CMatchFinderMT::~CMatchFinderMT()
{
m_ExitEvent.Set();
m_Thread.Wait();
FreeMem();
}
void CMatchFinderMT::Start()
{
m_AskChangeBufferPos.Reset();
m_CanChangeBufferPos.Reset();
m_BufferPosWasChanged.Reset();
m_WriteBufferIndex = 0;
m_ReadBufferIndex = 0;
m_NeedStart = false;
m_CurrentPos = 0;
m_CurrentLimitPos = 0;
m_Result = S_OK;
int i;
for (i = 0; i < kNumMTBlocks; i++)
m_CanReadEvents[i].Reset();
for (i = kNumMTBlocks - 1; i >= 0; i--)
m_CanWriteEvents[i].Set();
}
STDMETHODIMP_(UInt32) CMatchFinderMT::GetLongestMatch(UInt32 *distances)
{
if (m_CurrentPos == m_CurrentLimitPos)
{
if (m_NeedStart)
Start();
bool needStartEvent = true;
m_MtCanStart.Lock();
HRESULT result = S_OK;
UInt32 blockIndex = 0;
while (true)
{
HANDLE events[2] = { m_AskChangeBufferPos, m_CanReadEvents[m_ReadBufferIndex] } ;
DWORD waitResult = ::WaitForMultipleObjects(2, events, FALSE, INFINITE);
if (waitResult == WAIT_OBJECT_0 + 1)
m_CS[blockIndex].Enter();
if (needStartEvent)
{
m_MtWasStarted.Set();
needStartEvent = false;
}
else
m_CS[(blockIndex == 0) ? kNumMTBlocks - 1 : blockIndex - 1].Leave();
if (m_Exit)
return 0;
if (m_StopWriting)
{
m_MtWasStopped.Set();
m_CS[blockIndex].Leave();
break;
m_BufferPosWasChanged.Reset();
m_CanChangeBufferPos.Set();
m_BufferPosWasChanged.Lock();
}
if (result == S_OK)
{
IMatchFinder *mf = m_MatchFinder;
if (mf->NeedChangeBufferPos(kBlockSize) != 0)
{
// m_AskChangeBufferPos.Set();
m_StopReading[blockIndex] = true;
m_CS[blockIndex].Leave();
m_CanChangeBufferPos.Lock();
m_CS[blockIndex].Enter();
const Byte *bufferPosBefore = mf->GetPointerToCurrentPos();
mf->ChangeBufferPos();
m_DataCurrentPos += mf->GetPointerToCurrentPos() - bufferPosBefore;
m_BufferPosWasChanged.Set();
}
else
{
UInt32 curPos = blockIndex * kBlockSize;
UInt32 limit = curPos + kBlockSize - m_MatchMaxLen - m_MatchMaxLen - 1;
UInt32 *buffer = m_Buffer;
m_Results[blockIndex] = S_OK;
curPos++;
UInt32 numAvailableBytes = mf->GetNumAvailableBytes();
buffer[curPos++] = numAvailableBytes;
while (numAvailableBytes-- != 0 && curPos < limit)
{
result = mf->GetMatches(buffer + curPos);
if (result != S_OK)
{
m_Results[blockIndex] = result;
break;
}
curPos += buffer[curPos] + 1;
}
buffer[blockIndex * kBlockSize] = curPos;
}
}
else
{
UInt32 curPos = blockIndex * kBlockSize;
m_Buffer[curPos] = curPos + 2; // size of buffer
m_Buffer[curPos + 1] = 0; // NumAvailableBytes
m_Results[blockIndex] = result; // error
}
if (++blockIndex == kNumMTBlocks)
blockIndex = 0;
}
m_CurrentLimitPos = m_LimitPos[m_ReadBufferIndex];
m_NumAvailableBytesCurrent = m_NumAvailableBytes[m_ReadBufferIndex];
m_CurrentPos = 0;
m_Result = m_Results[m_ReadBufferIndex];
}
/*
if (m_CurrentPos >= m_CurrentLimitPos)
throw 1123324;
*/
const UInt32 *buffer = m_Buffers[m_ReadBufferIndex];
UInt32 len = buffer[m_CurrentPos++];
for (UInt32 i = 1; i <= len; i++)
distances[i] = buffer[m_CurrentPos++];
if (m_CurrentPos == m_CurrentLimitPos)
{
m_CanWriteEvents[m_ReadBufferIndex].Set();
if (++m_ReadBufferIndex == kNumMTBlocks)
m_ReadBufferIndex = 0;
}
return len;
}
STDMETHODIMP_(void) CMatchFinderMT::DummyLongestMatch()
{
GetLongestMatch(m_DummyBuffer);
}

View File

@@ -11,7 +11,7 @@
#include "../../../ICoder.h"
#include "../IMatchFinder.h"
const int kNumMTBlocks = 3;
const UInt32 kNumMTBlocks = (1 << 6);
class CMatchFinderMT:
public IMatchFinder,
@@ -19,65 +19,61 @@ class CMatchFinderMT:
{
MY_UNKNOWN_IMP
STDMETHOD(Init)(ISequentialInStream *s);
STDMETHOD(SetStream)(ISequentialInStream *inStream);
STDMETHOD_(void, ReleaseStream)();
STDMETHOD(MovePos)();
STDMETHOD(Init)();
STDMETHOD_(Byte, GetIndexByte)(Int32 index);
STDMETHOD_(UInt32, GetMatchLen)(Int32 index, UInt32 distance, UInt32 limit);
STDMETHOD_(UInt32, GetNumAvailableBytes)();
STDMETHOD_(const Byte *, GetPointerToCurrentPos)();
STDMETHOD(Create)(UInt32 sizeHistory,
UInt32 keepAddBufferBefore, UInt32 matchMaxLen,
UInt32 keepAddBufferAfter);
STDMETHOD_(UInt32, GetLongestMatch)(UInt32 *distances);
STDMETHOD_(void, DummyLongestMatch)();
STDMETHOD(Create)(UInt32 sizeHistory, UInt32 keepAddBufferBefore,
UInt32 matchMaxLen, UInt32 keepAddBufferAfter);
STDMETHOD(GetMatches)(UInt32 *distances);
STDMETHOD(Skip)(UInt32 num);
UInt32 m_CurrentPos;
UInt32 m_CurrentLimitPos;
STDMETHOD_(Int32, NeedChangeBufferPos)(UInt32 numCheckBytes);
STDMETHOD_(void, ChangeBufferPos)();
UInt32 m_Pos;
UInt32 m_PosLimit;
UInt32 m_MatchMaxLen;
UInt32 m_BlockSize;
UInt32 *m_Buffer;
UInt32 *m_Buffers[kNumMTBlocks];
UInt32 *m_DummyBuffer;
bool m_NeedStart;
UInt32 m_WriteBufferIndex;
UInt32 m_ReadBufferIndex;
NWindows::NSynchronization::CAutoResetEvent m_StopWriting;
NWindows::NSynchronization::CAutoResetEvent m_WritingWasStopped;
NWindows::NSynchronization::CManualResetEvent m_ExitEvent;
NWindows::NSynchronization::CAutoResetEvent m_CanReadEvents[kNumMTBlocks];
NWindows::NSynchronization::CAutoResetEvent m_CanWriteEvents[kNumMTBlocks];
HRESULT m_Results[kNumMTBlocks];
UInt32 m_LimitPos[kNumMTBlocks];
UInt32 m_NumAvailableBytes[kNumMTBlocks];
UInt32 m_NumAvailableBytesCurrent;
NWindows::CThread m_Thread;
UInt32 _multiThreadMult;
UInt32 m_BlockIndex;
HRESULT m_Result;
UInt32 m_NumAvailableBytes;
const Byte *m_DataCurrentPos;
void Start();
void FreeMem();
// Common variables
public:
NWindows::NSynchronization::CAutoResetEvent m_AskChangeBufferPos;
CMyComPtr<IMatchFinder> m_MatchFinder;
NWindows::CThread m_Thread;
NWindows::NSynchronization::CAutoResetEvent m_MtCanStart;
NWindows::NSynchronization::CAutoResetEvent m_MtWasStarted;
NWindows::NSynchronization::CAutoResetEvent m_MtWasStopped;
NWindows::NSynchronization::CAutoResetEvent m_CanChangeBufferPos;
NWindows::NSynchronization::CAutoResetEvent m_BufferPosWasChanged;
CMyComPtr<IMatchFinder> m_MatchFinder;
const Byte *m_DataCurrentPos;
NWindows::NSynchronization::CCriticalSection m_CS[kNumMTBlocks];
HRESULT m_Results[kNumMTBlocks];
bool m_StopReading[kNumMTBlocks];
bool m_Exit;
bool m_StopWriting;
////////////////////////////
void FreeMem();
void GetNextBlock();
public:
DWORD ThreadFunc();
CMatchFinderMT();
~CMatchFinderMT();
HRESULT SetMatchFinder(IMatchFinder *matchFinder, UInt32 multiThreadMult = 200);
HRESULT SetMatchFinder(IMatchFinder *matchFinder);
};
#endif