xevion/easy7zip
1
0
Fork 0
mirror of https://github.com/Xevion/easy7zip.git synced 2025-12-15 00:11:40 -06:00
Code Issues Packages Projects Releases Wiki Activity

4.33 beta

Browse Source
This commit is contained in:
Igor Pavlov
2006-02-05 00:00:00 +00:00
committed by Kornel Lesiński
parent e8d0636d7a
commit 02516d3fce
80 changed files with 2185 additions and 3957 deletions
Download Patch File Download Diff File Expand all files Collapse all files

32
7zip/Compress/LZ/BinTree/BinTree.h
View File

@@ -8,9 +8,8 @@ namespace BT_NAMESPACE {
typedef UInt32 CIndex;
const UInt32 kMaxValForNormalize = (UInt32(1) << 31) - 1;
class CMatchFinderBinTree:
class CMatchFinder:
public IMatchFinder,
public IMatchFinderSetCallback,
public CLZInWindow,
public CMyUnknownImp
{
@@ -18,37 +17,36 @@ class CMatchFinderBinTree:
UInt32 _cyclicBufferSize; // it must be historySize + 1
UInt32 _matchMaxLen;
CIndex *_hash;
CIndex *_son;
UInt32 _hashMask;
UInt32 _cutValue;
CMyComPtr<IMatchFinderCallback> m_Callback;
UInt32 _hashSizeSum;
void Normalize();
void FreeThisClassMemory();
void FreeMemory();
MY_UNKNOWN_IMP1(IMatchFinderSetCallback)
MY_UNKNOWN_IMP
STDMETHOD(Init)(ISequentialInStream *inStream);
STDMETHOD(SetStream)(ISequentialInStream *inStream);
STDMETHOD_(void, ReleaseStream)();
STDMETHOD(MovePos)();
STDMETHOD(Init)();
HRESULT MovePos();
STDMETHOD_(Byte, GetIndexByte)(Int32 index);
STDMETHOD_(UInt32, GetMatchLen)(Int32 index, UInt32 back, UInt32 limit);
STDMETHOD_(UInt32, GetNumAvailableBytes)();
STDMETHOD_(const Byte *, GetPointerToCurrentPos)();
STDMETHOD_(Int32, NeedChangeBufferPos)(UInt32 numCheckBytes);
STDMETHOD_(void, ChangeBufferPos)();
STDMETHOD(Create)(UInt32 historySize, UInt32 keepAddBufferBefore,
UInt32 matchMaxLen, UInt32 keepAddBufferAfter);
STDMETHOD_(UInt32, GetLongestMatch)(UInt32 *distances);
STDMETHOD_(void, DummyLongestMatch)();
// IMatchFinderSetCallback
STDMETHOD(SetCallback)(IMatchFinderCallback *callback);
virtual void BeforeMoveBlock();
virtual void AfterMoveBlock();
STDMETHOD(GetMatches)(UInt32 *distances);
STDMETHOD(Skip)(UInt32 num);
public:
CMatchFinderBinTree();
virtual ~CMatchFinderBinTree();
CMatchFinder();
virtual ~CMatchFinder();
void SetCutValue(UInt32 cutValue) { _cutValue = cutValue; }
};

4
7zip/Compress/LZ/BinTree/BinTree2.h
View File

@@ -3,10 +3,10 @@
#ifndef __BINTREE2_H
#define __BINTREE2_H
#undef BT_NAMESPACE
#define BT_NAMESPACE NBT2
#include "BinTree.h"
#include "BinTreeMain.h"
#undef BT_NAMESPACE
#endif

4
7zip/Compress/LZ/BinTree/BinTree3.h
View File

@@ -3,14 +3,14 @@
#ifndef __BINTREE3_H
#define __BINTREE3_H
#undef BT_NAMESPACE
#define BT_NAMESPACE NBT3
#define HASH_ARRAY_2
#include "BinTree.h"
#include "BinTreeMain.h"
#undef HASH_ARRAY_2
#undef BT_NAMESPACE
#endif

4
7zip/Compress/LZ/BinTree/BinTree3Z.h
View File

@@ -3,14 +3,14 @@
#ifndef __BINTREE3Z_H
#define __BINTREE3Z_H
#undef BT_NAMESPACE
#define BT_NAMESPACE NBT3Z
#define HASH_ZIP
#include "BinTree.h"
#include "BinTreeMain.h"
#undef HASH_ZIP
#undef BT_NAMESPACE
#endif

4
7zip/Compress/LZ/BinTree/BinTree4.h
View File

@@ -3,16 +3,16 @@
#ifndef __BINTREE4_H
#define __BINTREE4_H
#undef BT_NAMESPACE
#define BT_NAMESPACE NBT4
#define HASH_ARRAY_2
#define HASH_ARRAY_3
#include "BinTree.h"
#include "BinTreeMain.h"
#undef HASH_ARRAY_2
#undef HASH_ARRAY_3
#undef BT_NAMESPACE
#endif

20
7zip/Compress/LZ/BinTree/BinTree4b.h
View File

@@ -1,20 +0,0 @@
// BinTree4b.h
#ifndef __BINTREE4B_H
#define __BINTREE4B_H
#undef BT_NAMESPACE
#define BT_NAMESPACE NBT4B
#define HASH_ARRAY_2
#define HASH_ARRAY_3
#define HASH_BIG
#include "BinTree.h"
#include "BinTreeMain.h"
#undef HASH_ARRAY_2
#undef HASH_ARRAY_3
#undef HASH_BIG
#endif

523
7zip/Compress/LZ/BinTree/BinTreeMain.h
View File

@@ -4,38 +4,49 @@
#include "../../../../Common/CRC.h"
#include "../../../../Common/Alloc.h"
#include "BinTree.h"
// #include <xmmintrin.h>
// It's for prefetch
// But prefetch doesn't give big gain in K8.
namespace BT_NAMESPACE {
#ifdef HASH_ARRAY_2
static const UInt32 kHash2Size = 1 << 10;
#define kNumHashDirectBytes 0
#ifdef HASH_ARRAY_3
static const UInt32 kNumHashDirectBytes = 0;
static const UInt32 kNumHashBytes = 4;
static const UInt32 kHash3Size = 1 << 18;
#ifdef HASH_BIG
static const UInt32 kHashSize = 1 << 23;
#else
static const UInt32 kHashSize = 1 << 20;
#endif
static const UInt32 kHash3Size = 1 << 16;
#else
static const UInt32 kNumHashDirectBytes = 3;
static const UInt32 kNumHashBytes = 3;
static const UInt32 kHashSize = 1 << (8 * kNumHashBytes);
#endif
static const UInt32 kHashSize = 0;
static const UInt32 kMinMatchCheck = kNumHashBytes;
static const UInt32 kStartMaxLen = 1;
#else
#ifdef HASH_ZIP
static const UInt32 kNumHashDirectBytes = 0;
#define kNumHashDirectBytes 0
static const UInt32 kNumHashBytes = 3;
static const UInt32 kHashSize = 1 << 16;
static const UInt32 kMinMatchCheck = kNumHashBytes;
static const UInt32 kStartMaxLen = 1;
#else
#define THERE_ARE_DIRECT_HASH_BYTES
static const UInt32 kNumHashDirectBytes = 2;
#define kNumHashDirectBytes 2
static const UInt32 kNumHashBytes = 2;
static const UInt32 kHashSize = 1 << (8 * kNumHashBytes);
static const UInt32 kMinMatchCheck = kNumHashBytes + 1;
static const UInt32 kStartMaxLen = 1;
#endif
#endif
static const UInt32 kHashSizeSum = kHashSize
#ifdef HASH_ARRAY_2
#ifdef HASH_ARRAY_3
static const UInt32 kHash3Offset = kHash2Size;
#endif
#endif
static const UInt32 kFixHashSize = 0
#ifdef HASH_ARRAY_2
+ kHash2Size
#ifdef HASH_ARRAY_3
@@ -44,56 +55,86 @@ static const UInt32 kHashSizeSum = kHashSize
#endif
;
#ifdef HASH_ARRAY_2
static const UInt32 kHash2Offset = kHashSize;
#ifdef HASH_ARRAY_3
static const UInt32 kHash3Offset = kHashSize + kHash2Size;
#endif
#endif
CMatchFinderBinTree::CMatchFinderBinTree():
_hash(0),
_cutValue(0xFF)
CMatchFinder::CMatchFinder():
_hash(0)
{
}
void CMatchFinderBinTree::FreeThisClassMemory()
void CMatchFinder::FreeThisClassMemory()
{
BigFree(_hash);
_hash = 0;
}
void CMatchFinderBinTree::FreeMemory()
void CMatchFinder::FreeMemory()
{
FreeThisClassMemory();
CLZInWindow::Free();
}
CMatchFinderBinTree::~CMatchFinderBinTree()
CMatchFinder::~CMatchFinder()
{
FreeMemory();
}
STDMETHODIMP CMatchFinderBinTree::Create(UInt32 historySize, UInt32 keepAddBufferBefore,
STDMETHODIMP CMatchFinder::Create(UInt32 historySize, UInt32 keepAddBufferBefore,
UInt32 matchMaxLen, UInt32 keepAddBufferAfter)
{
if (historySize > kMaxValForNormalize - 256)
{
FreeMemory();
return E_INVALIDARG;
}
_cutValue =
#ifdef _HASH_CHAIN
8 + (matchMaxLen >> 2);
#else
16 + (matchMaxLen >> 1);
#endif
UInt32 sizeReserv = (historySize + keepAddBufferBefore +
matchMaxLen + keepAddBufferAfter) / 2 + 256;
if (CLZInWindow::Create(historySize + keepAddBufferBefore,
matchMaxLen + keepAddBufferAfter, sizeReserv))
{
if (historySize + 256 > kMaxValForNormalize)
{
FreeMemory();
return E_INVALIDARG;
}
_matchMaxLen = matchMaxLen;
UInt32 newCyclicBufferSize = historySize + 1;
if (_hash != 0 && newCyclicBufferSize == _cyclicBufferSize)
return S_OK;
FreeThisClassMemory();
_cyclicBufferSize = newCyclicBufferSize; // don't change it
_hash = (CIndex *)BigAlloc((kHashSizeSum + _cyclicBufferSize * 2) * sizeof(CIndex));
UInt32 hs = kHashSize;
#ifdef HASH_ARRAY_2
hs = historySize - 1;
hs |= (hs >> 1);
hs |= (hs >> 2);
hs |= (hs >> 4);
hs |= (hs >> 8);
hs >>= 1;
hs |= 0xFFFF;
if (hs > (1 << 24))
{
#ifdef HASH_ARRAY_3
hs >>= 1;
#else
hs = (1 << 24) - 1;
#endif
}
_hashMask = hs;
hs++;
#endif
_hashSizeSum = hs + kFixHashSize;
UInt32 numItems = _hashSizeSum + _cyclicBufferSize
#ifndef _HASH_CHAIN
* 2
#endif
;
size_t sizeInBytes = (size_t)numItems * sizeof(CIndex);
if (sizeInBytes / sizeof(CIndex) != numItems)
return E_OUTOFMEMORY;
_hash = (CIndex *)BigAlloc(sizeInBytes);
_son = _hash + _hashSizeSum;
if (_hash != 0)
return S_OK;
}
@@ -103,44 +144,47 @@ STDMETHODIMP CMatchFinderBinTree::Create(UInt32 historySize, UInt32 keepAddBuffe
static const UInt32 kEmptyHashValue = 0;
STDMETHODIMP CMatchFinderBinTree::Init(ISequentialInStream *stream)
STDMETHODIMP CMatchFinder::SetStream(ISequentialInStream *stream)
{
RINOK(CLZInWindow::Init(stream));
for(UInt32 i = 0; i < kHashSizeSum; i++)
CLZInWindow::SetStream(stream);
return S_OK;
}
STDMETHODIMP CMatchFinder::Init()
{
RINOK(CLZInWindow::Init());
for(UInt32 i = 0; i < _hashSizeSum; i++)
_hash[i] = kEmptyHashValue;
_cyclicBufferPos = 0;
ReduceOffsets(-1);
return S_OK;
}
STDMETHODIMP_(void) CMatchFinderBinTree::ReleaseStream()
STDMETHODIMP_(void) CMatchFinder::ReleaseStream()
{
// ReleaseStream();
}
#ifdef HASH_ARRAY_2
#ifdef HASH_ARRAY_3
inline UInt32 Hash(const Byte *pointer, UInt32 &hash2Value, UInt32 &hash3Value)
{
UInt32 temp = CCRC::Table[pointer[0]] ^ pointer[1];
hash2Value = temp & (kHash2Size - 1);
hash3Value = (temp ^ (UInt32(pointer[2]) << 8)) & (kHash3Size - 1);
return (temp ^ (UInt32(pointer[2]) << 8) ^ (CCRC::Table[pointer[3]] << 5)) &
(kHashSize - 1);
}
#define HASH_CALC { \
UInt32 temp = CCRC::Table[cur[0]] ^ cur[1]; \
hash2Value = temp & (kHash2Size - 1); \
hash3Value = (temp ^ (UInt32(cur[2]) << 8)) & (kHash3Size - 1); \
hashValue = (temp ^ (UInt32(cur[2]) << 8) ^ (CCRC::Table[cur[3]] << 5)) & _hashMask; }
#else // no HASH_ARRAY_3
inline UInt32 Hash(const Byte *pointer, UInt32 &hash2Value)
{
hash2Value = (CCRC::Table[pointer[0]] ^ pointer[1]) & (kHash2Size - 1);
return ((UInt32(pointer[0]) << 16)) | ((UInt32(pointer[1]) << 8)) | pointer[2];
}
#define HASH_CALC { \
UInt32 temp = CCRC::Table[cur[0]] ^ cur[1]; \
hash2Value = temp & (kHash2Size - 1); \
hashValue = (temp ^ (UInt32(cur[2]) << 8)) & _hashMask; }
#endif // HASH_ARRAY_3
#else // no HASH_ARRAY_2
#ifdef HASH_ZIP
inline UInt32 Hash(const Byte *pointer)
{
return ((UInt32(pointer[0]) << 8) ^
CCRC::Table[pointer[1]] ^ pointer[2]) & (kHashSize - 1);
return ((UInt32(pointer[0]) << 8) ^ CCRC::Table[pointer[1]] ^ pointer[2]) & (kHashSize - 1);
}
#else // no HASH_ZIP
inline UInt32 Hash(const Byte *pointer)
@@ -150,7 +194,7 @@ inline UInt32 Hash(const Byte *pointer)
#endif // HASH_ZIP
#endif // HASH_ARRAY_2
STDMETHODIMP_(UInt32) CMatchFinderBinTree::GetLongestMatch(UInt32 *distances)
STDMETHODIMP CMatchFinder::GetMatches(UInt32 *distances)
{
UInt32 lenLimit;
if (_pos + _matchMaxLen <= _streamPos)
@@ -158,233 +202,284 @@ STDMETHODIMP_(UInt32) CMatchFinderBinTree::GetLongestMatch(UInt32 *distances)
else
{
lenLimit = _streamPos - _pos;
if(lenLimit < kNumHashBytes)
return 0;
if(lenLimit < kMinMatchCheck)
{
distances[0] = 0;
return MovePos();
}
}
UInt32 matchMinPos = (_pos > _cyclicBufferSize) ? (_pos - _cyclicBufferSize) : 0;
Byte *cur = _buffer + _pos;
int offset = 1;
UInt32 maxLen = 0;
UInt32 matchMinPos = (_pos > _cyclicBufferSize) ? (_pos - _cyclicBufferSize) : 0;
const Byte *cur = _buffer + _pos;
UInt32 maxLen = kStartMaxLen; // to avoid items for len < hashSize;
#ifdef HASH_ARRAY_2
UInt32 hash2Value;
#ifdef HASH_ARRAY_3
UInt32 hash3Value;
UInt32 hashValue = Hash(cur, hash2Value, hash3Value);
#else
UInt32 hashValue = Hash(cur, hash2Value);
#endif
UInt32 hashValue;
HASH_CALC;
#else
UInt32 hashValue = Hash(cur);
#endif
UInt32 curMatch = _hash[hashValue];
UInt32 curMatch = _hash[kFixHashSize + hashValue];
#ifdef HASH_ARRAY_2
UInt32 curMatch2 = _hash[kHash2Offset + hash2Value];
UInt32 curMatch2 = _hash[hash2Value];
#ifdef HASH_ARRAY_3
UInt32 curMatch3 = _hash[kHash3Offset + hash3Value];
#endif
_hash[kHash2Offset + hash2Value] = _pos;
distances[2] = 0xFFFFFFFF;
_hash[hash2Value] = _pos;
if(curMatch2 > matchMinPos)
if (_buffer[curMatch2] == cur[0])
{
distances[2] = _pos - curMatch2 - 1;
maxLen = 2;
distances[offset++] = maxLen = 2;
distances[offset++] = _pos - curMatch2 - 1;
}
#ifdef HASH_ARRAY_3
_hash[kHash3Offset + hash3Value] = _pos;
distances[3] = 0xFFFFFFFF;
if(curMatch3 > matchMinPos)
if (_buffer[curMatch3] == cur[0])
{
distances[3] = _pos - curMatch3 - 1;
maxLen = 3;
if (curMatch3 == curMatch2)
offset -= 2;
distances[offset++] = maxLen = 3;
distances[offset++] = _pos - curMatch3 - 1;
curMatch2 = curMatch3;
}
#endif
if (offset != 1 && curMatch2 == curMatch)
{
offset -= 2;
maxLen = kStartMaxLen;
}
#endif
_hash[hashValue] = _pos;
_hash[kFixHashSize + hashValue] = _pos;
CIndex *son = _hash + kHashSizeSum;
CIndex *son = _son;
#ifdef _HASH_CHAIN
son[_cyclicBufferPos] = curMatch;
#else
CIndex *ptr0 = son + (_cyclicBufferPos << 1) + 1;
CIndex *ptr1 = son + (_cyclicBufferPos << 1);
distances[kNumHashBytes] = 0xFFFFFFFF;
#ifdef THERE_ARE_DIRECT_HASH_BYTES
if (lenLimit == kNumHashDirectBytes)
{
if(curMatch > matchMinPos)
while (maxLen < kNumHashDirectBytes)
distances[++maxLen] = _pos - curMatch - 1;
// We don't need tree in this case
}
else
UInt32 len0, len1;
len0 = len1 = kNumHashDirectBytes;
#endif
#if kNumHashDirectBytes != 0
if(curMatch > matchMinPos)
{
UInt32 len0, len1;
len0 = len1 = kNumHashDirectBytes;
UInt32 count = _cutValue;
while(true)
if (_buffer[curMatch + kNumHashDirectBytes] != cur[kNumHashDirectBytes])
{
if(curMatch <= matchMinPos || count-- == 0)
{
*ptr0 = kEmptyHashValue;
*ptr1 = kEmptyHashValue;
break;
}
Byte *pb = _buffer + curMatch;
UInt32 len = MyMin(len0, len1);
do
{
if (pb[len] != cur[len])
break;
}
while(++len != lenLimit);
UInt32 delta = _pos - curMatch;
while (maxLen < len)
distances[++maxLen] = delta - 1;
UInt32 cyclicPos = (delta <= _cyclicBufferPos) ?
(_cyclicBufferPos - delta):
(_cyclicBufferPos - delta + _cyclicBufferSize);
CIndex *pair = son + (cyclicPos << 1);
if (len != lenLimit)
{
if (pb[len] < cur[len])
{
*ptr1 = curMatch;
ptr1 = pair + 1;
curMatch = *ptr1;
len1 = len;
}
else
{
*ptr0 = curMatch;
ptr0 = pair;
curMatch = *ptr0;
len0 = len;
}
}
else
{
*ptr1 = pair[0];
*ptr0 = pair[1];
break;
}
distances[offset++] = maxLen = kNumHashDirectBytes;
distances[offset++] = _pos - curMatch - 1;
}
}
#ifdef HASH_ARRAY_2
#ifdef HASH_ARRAY_3
if (distances[4] < distances[3])
distances[3] = distances[4];
#endif
if (distances[3] < distances[2])
distances[2] = distances[3];
#endif
return maxLen;
UInt32 count = _cutValue;
while(true)
{
if(curMatch <= matchMinPos || count-- == 0)
{
#ifndef _HASH_CHAIN
*ptr0 = *ptr1 = kEmptyHashValue;
#endif
break;
}
UInt32 delta = _pos - curMatch;
UInt32 cyclicPos = (delta <= _cyclicBufferPos) ?
(_cyclicBufferPos - delta):
(_cyclicBufferPos - delta + _cyclicBufferSize);
CIndex *pair = son +
#ifdef _HASH_CHAIN
cyclicPos;
#else
(cyclicPos << 1);
#endif
// _mm_prefetch((const char *)pair, _MM_HINT_T0);
const Byte *pb = _buffer + curMatch;
UInt32 len =
#ifdef _HASH_CHAIN
kNumHashDirectBytes;
if (pb[maxLen] == cur[maxLen])
#else
MyMin(len0, len1);
#endif
if (pb[len] == cur[len])
{
while(++len != lenLimit)
if (pb[len] != cur[len])
break;
if (maxLen < len)
{
distances[offset++] = maxLen = len;
distances[offset++] = delta - 1;
if (len == lenLimit)
{
#ifndef _HASH_CHAIN
*ptr1 = pair[0];
*ptr0 = pair[1];
#endif
break;
}
}
}
#ifdef _HASH_CHAIN
curMatch = *pair;
#else
if (pb[len] < cur[len])
{
*ptr1 = curMatch;
ptr1 = pair + 1;
curMatch = *ptr1;
len1 = len;
}
else
{
*ptr0 = curMatch;
ptr0 = pair;
curMatch = *ptr0;
len0 = len;
}
#endif
}
distances[0] = offset - 1;
if (++_cyclicBufferPos == _cyclicBufferSize)
_cyclicBufferPos = 0;
RINOK(CLZInWindow::MovePos());
if (_pos == kMaxValForNormalize)
Normalize();
return S_OK;
}
STDMETHODIMP_(void) CMatchFinderBinTree::DummyLongestMatch()
STDMETHODIMP CMatchFinder::Skip(UInt32 num)
{
do
{
#ifdef _HASH_CHAIN
if (_streamPos - _pos < kNumHashBytes)
{
RINOK(MovePos());
continue;
}
#else
UInt32 lenLimit;
if (_pos + _matchMaxLen <= _streamPos)
lenLimit = _matchMaxLen;
else
{
lenLimit = _streamPos - _pos;
if(lenLimit < kNumHashBytes)
return;
if(lenLimit < kMinMatchCheck)
{
RINOK(MovePos());
continue;
}
}
UInt32 matchMinPos = (_pos > _cyclicBufferSize) ? (_pos - _cyclicBufferSize) : 0;
Byte *cur = _buffer + _pos;
#endif
const Byte *cur = _buffer + _pos;
#ifdef HASH_ARRAY_2
UInt32 hash2Value;
#ifdef HASH_ARRAY_3
UInt32 hash3Value;
UInt32 hashValue = Hash(cur, hash2Value, hash3Value);
UInt32 hashValue;
HASH_CALC;
_hash[kHash3Offset + hash3Value] = _pos;
#else
UInt32 hashValue = Hash(cur, hash2Value);
UInt32 hashValue;
HASH_CALC;
#endif
_hash[kHash2Offset + hash2Value] = _pos;
_hash[hash2Value] = _pos;
#else
UInt32 hashValue = Hash(cur);
#endif
UInt32 curMatch = _hash[hashValue];
_hash[hashValue] = _pos;
UInt32 curMatch = _hash[kFixHashSize + hashValue];
_hash[kFixHashSize + hashValue] = _pos;
CIndex *son = _hash + kHashSizeSum;
#ifdef _HASH_CHAIN
_son[_cyclicBufferPos] = curMatch;
#else
CIndex *son = _son;
CIndex *ptr0 = son + (_cyclicBufferPos << 1) + 1;
CIndex *ptr1 = son + (_cyclicBufferPos << 1);
#ifdef THERE_ARE_DIRECT_HASH_BYTES
if (lenLimit != kNumHashDirectBytes)
#endif
UInt32 len0, len1;
len0 = len1 = kNumHashDirectBytes;
UInt32 count = _cutValue;
while(true)
{
UInt32 len0, len1;
len0 = len1 = kNumHashDirectBytes;
UInt32 count = _cutValue;
while(true)
if(curMatch <= matchMinPos || count-- == 0)
{
if(curMatch <= matchMinPos || count-- == 0)
break;
Byte *pb = _buffer + curMatch;
UInt32 len = MyMin(len0, len1);
do
{
*ptr0 = *ptr1 = kEmptyHashValue;
break;
}
UInt32 delta = _pos - curMatch;
UInt32 cyclicPos = (delta <= _cyclicBufferPos) ?
(_cyclicBufferPos - delta):
(_cyclicBufferPos - delta + _cyclicBufferSize);
CIndex *pair = son + (cyclicPos << 1);
// _mm_prefetch((const char *)pair, _MM_HINT_T0);
const Byte *pb = _buffer + curMatch;
UInt32 len = MyMin(len0, len1);
if (pb[len] == cur[len])
{
while(++len != lenLimit)
if (pb[len] != cur[len])
break;
}
while(++len != lenLimit);
UInt32 delta = _pos - curMatch;
UInt32 cyclicPos = (delta <= _cyclicBufferPos) ?
(_cyclicBufferPos - delta):
(_cyclicBufferPos - delta + _cyclicBufferSize);
CIndex *pair = son + (cyclicPos << 1);
if (len != lenLimit)
{
if (pb[len] < cur[len])
{
*ptr1 = curMatch;
ptr1 = pair + 1;
curMatch = *ptr1;
len1 = len;
}
else
{
*ptr0 = curMatch;
ptr0 = pair;
curMatch = *ptr0;
len0 = len;
}
}
else
if (len == lenLimit)
{
*ptr1 = pair[0];
*ptr0 = pair[1];
return;
break;
}
}
if (pb[len] < cur[len])
{
*ptr1 = curMatch;
ptr1 = pair + 1;
curMatch = *ptr1;
len1 = len;
}
else
{
*ptr0 = curMatch;
ptr0 = pair;
curMatch = *ptr0;
len0 = len;
}
}
*ptr0 = kEmptyHashValue;
*ptr1 = kEmptyHashValue;
#endif
if (++_cyclicBufferPos == _cyclicBufferSize)
_cyclicBufferPos = 0;
RINOK(CLZInWindow::MovePos());
if (_pos == kMaxValForNormalize)
Normalize();
}
while(--num != 0);
return S_OK;
}
void CMatchFinderBinTree::Normalize()
void CMatchFinder::Normalize()
{
UInt32 subValue = _pos - _cyclicBufferSize;
CIndex *items = _hash;
UInt32 numItems = (kHashSizeSum + _cyclicBufferSize * 2);
UInt32 numItems = (_hashSizeSum + _cyclicBufferSize * 2);
for (UInt32 i = 0; i < numItems; i++)
{
UInt32 value = items[i];
@@ -397,7 +492,7 @@ void CMatchFinderBinTree::Normalize()
ReduceOffsets(subValue);
}
STDMETHODIMP CMatchFinderBinTree::MovePos()
HRESULT CMatchFinder::MovePos()
{
if (++_cyclicBufferPos == _cyclicBufferSize)
_cyclicBufferPos = 0;
@@ -407,38 +502,26 @@ STDMETHODIMP CMatchFinderBinTree::MovePos()
return S_OK;
}
STDMETHODIMP_(Byte) CMatchFinderBinTree::GetIndexByte(Int32 index)
STDMETHODIMP_(Byte) CMatchFinder::GetIndexByte(Int32 index)
{ return CLZInWindow::GetIndexByte(index); }
STDMETHODIMP_(UInt32) CMatchFinderBinTree::GetMatchLen(Int32 index,
STDMETHODIMP_(UInt32) CMatchFinder::GetMatchLen(Int32 index,
UInt32 back, UInt32 limit)
{ return CLZInWindow::GetMatchLen(index, back, limit); }
STDMETHODIMP_(UInt32) CMatchFinderBinTree::GetNumAvailableBytes()
STDMETHODIMP_(UInt32) CMatchFinder::GetNumAvailableBytes()
{ return CLZInWindow::GetNumAvailableBytes(); }
STDMETHODIMP_(const Byte *) CMatchFinderBinTree::GetPointerToCurrentPos()
STDMETHODIMP_(const Byte *) CMatchFinder::GetPointerToCurrentPos()
{ return CLZInWindow::GetPointerToCurrentPos(); }
// IMatchFinderSetCallback
STDMETHODIMP CMatchFinderBinTree::SetCallback(IMatchFinderCallback *callback)
{
m_Callback = callback;
return S_OK;
}
STDMETHODIMP_(Int32) CMatchFinder::NeedChangeBufferPos(UInt32 numCheckBytes)
{ return CLZInWindow::NeedMove(numCheckBytes) ? 1: 0; }
void CMatchFinderBinTree::BeforeMoveBlock()
{
if (m_Callback)
m_Callback->BeforeChangingBufferPos();
CLZInWindow::BeforeMoveBlock();
}
STDMETHODIMP_(void) CMatchFinder::ChangeBufferPos()
{ CLZInWindow::MoveBlock();}
void CMatchFinderBinTree::AfterMoveBlock()
{
if (m_Callback)
m_Callback->AfterChangingBufferPos();
CLZInWindow::AfterMoveBlock();
}
#undef HASH_CALC
#undef kNumHashDirectBytes
}

55
7zip/Compress/LZ/HashChain/HC.h
View File

@@ -1,55 +0,0 @@
// HC.h
#include "../LZInWindow.h"
#include "../IMatchFinder.h"
namespace HC_NAMESPACE {
typedef UInt32 CIndex;
const UInt32 kMaxValForNormalize = (UInt32(1) << 31) - 1;
class CMatchFinderHC:
public IMatchFinder,
public IMatchFinderSetCallback,
public CLZInWindow,
public CMyUnknownImp
{
UInt32 _cyclicBufferPos;
UInt32 _cyclicBufferSize; // it must be historySize + 1
UInt32 _matchMaxLen;
CIndex *_hash;
UInt32 _cutValue;
CMyComPtr<IMatchFinderCallback> m_Callback;
void Normalize();
void FreeThisClassMemory();
void FreeMemory();
MY_UNKNOWN_IMP1(IMatchFinderSetCallback)
STDMETHOD(Init)(ISequentialInStream *inStream);
STDMETHOD_(void, ReleaseStream)();
STDMETHOD(MovePos)();
STDMETHOD_(Byte, GetIndexByte)(Int32 index);
STDMETHOD_(UInt32, GetMatchLen)(Int32 index, UInt32 back, UInt32 limit);
STDMETHOD_(UInt32, GetNumAvailableBytes)();
STDMETHOD_(const Byte *, GetPointerToCurrentPos)();
STDMETHOD(Create)(UInt32 historySize, UInt32 keepAddBufferBefore,
UInt32 matchMaxLen, UInt32 keepAddBufferAfter);
STDMETHOD_(UInt32, GetLongestMatch)(UInt32 *distances);
STDMETHOD_(void, DummyLongestMatch)();
// IMatchFinderSetCallback
STDMETHOD(SetCallback)(IMatchFinderCallback *callback);
virtual void BeforeMoveBlock();
virtual void AfterMoveBlock();
public:
CMatchFinderHC();
virtual ~CMatchFinderHC();
void SetCutValue(UInt32 cutValue) { _cutValue = cutValue; }
};
}

8
7zip/Compress/LZ/HashChain/HC2.h
View File

@@ -3,11 +3,11 @@
#ifndef __HC2_H
#define __HC2_H
#undef HC_NAMESPACE
#define HC_NAMESPACE NHC2
#define BT_NAMESPACE NHC2
#include "HCMF.h"
#include "HCMFMain.h"
#include "HCMain.h"
#undef BT_NAMESPACE
#endif

5
7zip/Compress/LZ/HashChain/HC3.h
View File

@@ -3,15 +3,14 @@
#ifndef __HC3_H
#define __HC3_H
#undef HC_NAMESPACE
#define HC_NAMESPACE NHC3
#define BT_NAMESPACE NHC3
#define HASH_ARRAY_2
#include "HC.h"
#include "HCMain.h"
#undef HASH_ARRAY_2
#undef BT_NAMESPACE
#endif

6
7zip/Compress/LZ/HashChain/HC4.h
View File

@@ -3,17 +3,17 @@
#ifndef __HC4_H
#define __HC4_H
#undef HC_NAMESPACE
#define HC_NAMESPACE NHC4
#define BT_NAMESPACE NHC4
#define HASH_ARRAY_2
#define HASH_ARRAY_3
#include "HC.h"
#include "HCMain.h"
#undef HASH_ARRAY_2
#undef HASH_ARRAY_3
#undef BT_NAMESPACE
#endif

21
7zip/Compress/LZ/HashChain/HC4b.h
View File

@@ -1,21 +0,0 @@
// HC4b.h
#ifndef __HC4B__H
#define __HC4B__H
#undef HC_NAMESPACE
#define HC_NAMESPACE NHC4b
#define HASH_ARRAY_2
#define HASH_ARRAY_3
#define HASH_BIG
#include "HC.h"
#include "HCMain.h"
#undef HASH_ARRAY_2
#undef HASH_ARRAY_3
#undef HASH_BIG
#endif

352
7zip/Compress/LZ/HashChain/HCMain.h
View File

@@ -1,350 +1,6 @@
// HC.h
// HCMain.h
#include "../../../../Common/Defs.h"
#include "../../../../Common/CRC.h"
#include "../../../../Common/Alloc.h"
#define _HASH_CHAIN
#include "../BinTree/BinTreeMain.h"
#undef _HASH_CHAIN
namespace HC_NAMESPACE {
#ifdef HASH_ARRAY_2
static const UInt32 kHash2Size = 1 << 10;
#ifdef HASH_ARRAY_3
static const UInt32 kNumHashDirectBytes = 0;
static const UInt32 kNumHashBytes = 4;
static const UInt32 kHash3Size = 1 << 18;
#ifdef HASH_BIG
static const UInt32 kHashSize = 1 << 23;
#else
static const UInt32 kHashSize = 1 << 20;
#endif
#else
static const UInt32 kNumHashDirectBytes = 0;
static const UInt32 kNumHashBytes = 3;
static const UInt32 kHashSize = 1 << (16);
#endif
#else
#ifdef HASH_ZIP
static const UInt32 kNumHashDirectBytes = 0;
static const UInt32 kNumHashBytes = 3;
static const UInt32 kHashSize = 1 << 16;
#else
#define THERE_ARE_DIRECT_HASH_BYTES
static const UInt32 kNumHashDirectBytes = 2;
static const UInt32 kNumHashBytes = 2;
static const UInt32 kHashSize = 1 << (8 * kNumHashBytes);
#endif
#endif
static const UInt32 kHashSizeSum = kHashSize
#ifdef HASH_ARRAY_2
+ kHash2Size
#ifdef HASH_ARRAY_3
+ kHash3Size
#endif
#endif
;
#ifdef HASH_ARRAY_2
static const UInt32 kHash2Offset = kHashSize;
#ifdef HASH_ARRAY_3
static const UInt32 kHash3Offset = kHashSize + kHash2Size;
#endif
#endif
CMatchFinderHC::CMatchFinderHC():
_hash(0),
_cutValue(16)
{
}
void CMatchFinderHC::FreeThisClassMemory()
{
BigFree(_hash);
_hash = 0;
}
void CMatchFinderHC::FreeMemory()
{
FreeThisClassMemory();
CLZInWindow::Free();
}
CMatchFinderHC::~CMatchFinderHC()
{
FreeMemory();
}
STDMETHODIMP CMatchFinderHC::Create(UInt32 historySize, UInt32 keepAddBufferBefore,
UInt32 matchMaxLen, UInt32 keepAddBufferAfter)
{
UInt32 sizeReserv = (historySize + keepAddBufferBefore +
matchMaxLen + keepAddBufferAfter) / 2 + 256;
if (CLZInWindow::Create(historySize + keepAddBufferBefore,
matchMaxLen + keepAddBufferAfter, sizeReserv))
{
if (historySize + 256 > kMaxValForNormalize)
{
FreeMemory();
return E_INVALIDARG;
}
_matchMaxLen = matchMaxLen;
UInt32 newCyclicBufferSize = historySize + 1;
if (_hash != 0 && newCyclicBufferSize == _cyclicBufferSize)
return S_OK;
FreeThisClassMemory();
_cyclicBufferSize = newCyclicBufferSize; // don't change it
_hash = (CIndex *)BigAlloc((kHashSizeSum + _cyclicBufferSize) * sizeof(CIndex));
if (_hash != 0)
return S_OK;
}
FreeMemory();
return E_OUTOFMEMORY;
}
static const UInt32 kEmptyHashValue = 0;
STDMETHODIMP CMatchFinderHC::Init(ISequentialInStream *stream)
{
RINOK(CLZInWindow::Init(stream));
for(UInt32 i = 0; i < kHashSizeSum; i++)
_hash[i] = kEmptyHashValue;
_cyclicBufferPos = 0;
ReduceOffsets(-1);
return S_OK;
}
STDMETHODIMP_(void) CMatchFinderHC::ReleaseStream()
{
// ReleaseStream();
}
#ifdef HASH_ARRAY_2
#ifdef HASH_ARRAY_3
inline UInt32 Hash(const Byte *pointer, UInt32 &hash2Value, UInt32 &hash3Value)
{
UInt32 temp = CCRC::Table[pointer[0]] ^ pointer[1];
hash2Value = temp & (kHash2Size - 1);
hash3Value = (temp ^ (UInt32(pointer[2]) << 8)) & (kHash3Size - 1);
return (temp ^ (UInt32(pointer[2]) << 8) ^ (CCRC::Table[pointer[3]] << 5)) &
(kHashSize - 1);
}
#else // no HASH_ARRAY_3
inline UInt32 Hash(const Byte *pointer, UInt32 &hash2Value)
{
UInt32 temp = CCRC::Table[pointer[0]] ^ pointer[1];
hash2Value = temp & (kHash2Size - 1);
return (temp ^ (UInt32(pointer[2]) << 8)) & (kHashSize - 1);;
}
#endif // HASH_ARRAY_3
#else // no HASH_ARRAY_2
#ifdef HASH_ZIP
inline UInt32 Hash(const Byte *pointer)
{
return ((UInt32(pointer[0]) << 8) ^
CCRC::Table[pointer[1]] ^ pointer[2]) & (kHashSize - 1);
}
#else // no HASH_ZIP
inline UInt32 Hash(const Byte *pointer)
{
return pointer[0] ^ (UInt32(pointer[1]) << 8);
}
#endif // HASH_ZIP
#endif // HASH_ARRAY_2
STDMETHODIMP_(UInt32) CMatchFinderHC::GetLongestMatch(UInt32 *distances)
{
UInt32 lenLimit;
if (_pos + _matchMaxLen <= _streamPos)
lenLimit = _matchMaxLen;
else
{
lenLimit = _streamPos - _pos;
if(lenLimit < kNumHashBytes)
return 0;
}
UInt32 matchMinPos = (_pos > _cyclicBufferSize) ? (_pos - _cyclicBufferSize) : 0;
Byte *cur = _buffer + _pos;
UInt32 maxLen = 0;
#ifdef HASH_ARRAY_2
UInt32 hash2Value;
#ifdef HASH_ARRAY_3
UInt32 hash3Value;
UInt32 hashValue = Hash(cur, hash2Value, hash3Value);
#else
UInt32 hashValue = Hash(cur, hash2Value);
#endif
#else
UInt32 hashValue = Hash(cur);
#endif
#ifdef HASH_ARRAY_2
UInt32 curMatch2 = _hash[kHash2Offset + hash2Value];
_hash[kHash2Offset + hash2Value] = _pos;
distances[2] = 0xFFFFFFFF;
if(curMatch2 > matchMinPos)
if (_buffer[curMatch2] == cur[0])
{
distances[2] = _pos - curMatch2 - 1;
maxLen = 2;
}
#ifdef HASH_ARRAY_3
UInt32 curMatch3 = _hash[kHash3Offset + hash3Value];
_hash[kHash3Offset + hash3Value] = _pos;
distances[3] = 0xFFFFFFFF;
if(curMatch3 > matchMinPos)
if (_buffer[curMatch3] == cur[0])
{
distances[3] = _pos - curMatch3 - 1;
maxLen = 3;
}
#endif
#endif
UInt32 curMatch = _hash[hashValue];
_hash[hashValue] = _pos;
CIndex *chain = _hash + kHashSizeSum;
chain[_cyclicBufferPos] = curMatch;
distances[kNumHashBytes] = 0xFFFFFFFF;
#ifdef THERE_ARE_DIRECT_HASH_BYTES
if (lenLimit == kNumHashDirectBytes)
{
if(curMatch > matchMinPos)
while (maxLen < kNumHashDirectBytes)
distances[++maxLen] = _pos - curMatch - 1;
}
else
#endif
{
UInt32 count = _cutValue;
do
{
if(curMatch <= matchMinPos)
break;
Byte *pby1 = _buffer + curMatch;
UInt32 currentLen = kNumHashDirectBytes;
do
{
if (pby1[currentLen] != cur[currentLen])
break;
}
while(++currentLen != lenLimit);
UInt32 delta = _pos - curMatch;
while (maxLen < currentLen)
distances[++maxLen] = delta - 1;
if(currentLen == lenLimit)
break;
UInt32 cyclicPos = (delta <= _cyclicBufferPos) ?
(_cyclicBufferPos - delta):
(_cyclicBufferPos - delta + _cyclicBufferSize);
curMatch = chain[cyclicPos];
}
while(--count != 0);
}
#ifdef HASH_ARRAY_2
#ifdef HASH_ARRAY_3
if (distances[4] < distances[3])
distances[3] = distances[4];
#endif
if (distances[3] < distances[2])
distances[2] = distances[3];
#endif
return maxLen;
}
STDMETHODIMP_(void) CMatchFinderHC::DummyLongestMatch()
{
if (_streamPos - _pos < kNumHashBytes)
return;
Byte *cur = _buffer + _pos;
#ifdef HASH_ARRAY_2
UInt32 hash2Value;
#ifdef HASH_ARRAY_3
UInt32 hash3Value;
UInt32 hashValue = Hash(cur, hash2Value, hash3Value);
_hash[kHash3Offset + hash3Value] = _pos;
#else
UInt32 hashValue = Hash(cur, hash2Value);
#endif
_hash[kHash2Offset + hash2Value] = _pos;
#else
UInt32 hashValue = Hash(cur);
#endif
_hash[kHashSizeSum + _cyclicBufferPos] = _hash[hashValue];
_hash[hashValue] = _pos;
}
void CMatchFinderHC::Normalize()
{
UInt32 subValue = _pos - _cyclicBufferSize;
CIndex *items = _hash;
UInt32 numItems = kHashSizeSum + _cyclicBufferSize;
for (UInt32 i = 0; i < numItems; i++)
{
UInt32 value = items[i];
if (value <= subValue)
value = kEmptyHashValue;
else
value -= subValue;
items[i] = value;
}
ReduceOffsets(subValue);
}
STDMETHODIMP CMatchFinderHC::MovePos()
{
if (++_cyclicBufferPos == _cyclicBufferSize)
_cyclicBufferPos = 0;
RINOK(CLZInWindow::MovePos());
if (_pos == kMaxValForNormalize)
Normalize();
return S_OK;
}
STDMETHODIMP_(Byte) CMatchFinderHC::GetIndexByte(Int32 index)
{ return CLZInWindow::GetIndexByte(index); }
STDMETHODIMP_(UInt32) CMatchFinderHC::GetMatchLen(Int32 index,
UInt32 back, UInt32 limit)
{ return CLZInWindow::GetMatchLen(index, back, limit); }
STDMETHODIMP_(UInt32) CMatchFinderHC::GetNumAvailableBytes()
{ return CLZInWindow::GetNumAvailableBytes(); }
STDMETHODIMP_(const Byte *) CMatchFinderHC::GetPointerToCurrentPos()
{ return CLZInWindow::GetPointerToCurrentPos(); }
// IMatchFinderSetCallback
STDMETHODIMP CMatchFinderHC::SetCallback(IMatchFinderCallback *callback)
{
m_Callback = callback;
return S_OK;
}
void CMatchFinderHC::BeforeMoveBlock()
{
if (m_Callback)
m_Callback->BeforeChangingBufferPos();
CLZInWindow::BeforeMoveBlock();
}
void CMatchFinderHC::AfterMoveBlock()
{
if (m_Callback)
m_Callback->AfterChangingBufferPos();
CLZInWindow::AfterMoveBlock();
}
}

52
7zip/Compress/LZ/IMatchFinder.h
View File

@@ -3,61 +3,25 @@
#ifndef __IMATCHFINDER_H
#define __IMATCHFINDER_H
// {23170F69-40C1-278A-0000-000200010000}
DEFINE_GUID(IID_IInWindowStream,
0x23170F69, 0x40C1, 0x278A, 0x00, 0x00, 0x00, 0x02, 0x00, 0x01, 0x00, 0x00);
MIDL_INTERFACE("23170F69-40C1-278A-0000-000200010000")
IInWindowStream: public IUnknown
struct IInWindowStream: public IUnknown
{
STDMETHOD(Init)(ISequentialInStream *inStream) PURE;
STDMETHOD(SetStream)(ISequentialInStream *inStream) PURE;
STDMETHOD_(void, ReleaseStream)() PURE;
STDMETHOD(MovePos)() PURE;
STDMETHOD(Init)() PURE;
STDMETHOD_(Byte, GetIndexByte)(Int32 index) PURE;
STDMETHOD_(UInt32, GetMatchLen)(Int32 index, UInt32 distance, UInt32 limit) PURE;
STDMETHOD_(UInt32, GetNumAvailableBytes)() PURE;
STDMETHOD_(const Byte *, GetPointerToCurrentPos)() PURE;
STDMETHOD_(Int32, NeedChangeBufferPos)(UInt32 numCheckBytes) PURE;
STDMETHOD_(void, ChangeBufferPos)() PURE;
};
// {23170F69-40C1-278A-0000-000200020000}
DEFINE_GUID(IID_IMatchFinder,
0x23170F69, 0x40C1, 0x278A, 0x00, 0x00, 0x00, 0x02, 0x00, 0x02, 0x00, 0x00);
MIDL_INTERFACE("23170F69-40C1-278A-0000-000200020000")
IMatchFinder: public IInWindowStream
struct IMatchFinder: public IInWindowStream
{
STDMETHOD(Create)(UInt32 historySize, UInt32 keepAddBufferBefore,
UInt32 matchMaxLen, UInt32 keepAddBufferAfter) PURE;
STDMETHOD_(UInt32, GetLongestMatch)(UInt32 *distances) PURE;
STDMETHOD_(void, DummyLongestMatch)() PURE;
STDMETHOD(GetMatches)(UInt32 *distances) PURE;
STDMETHOD(Skip)(UInt32 num) PURE;
};
// {23170F69-40C1-278A-0000-000200020100}
DEFINE_GUID(IID_IMatchFinderCallback,
0x23170F69, 0x40C1, 0x278A, 0x00, 0x00, 0x00, 0x02, 0x00, 0x02, 0x01, 0x00);
MIDL_INTERFACE("23170F69-40C1-278A-0000-000200020100")
IMatchFinderCallback: public IUnknown
{
STDMETHOD(BeforeChangingBufferPos)() PURE;
STDMETHOD(AfterChangingBufferPos)() PURE;
};
// {23170F69-40C1-278A-0000-000200020200}
DEFINE_GUID(IID_IMatchFinderSetCallback,
0x23170F69, 0x40C1, 0x278A, 0x00, 0x00, 0x00, 0x02, 0x00, 0x02, 0x02, 0x00);
MIDL_INTERFACE("23170F69-40C1-278A-0000-000200020200")
IMatchFinderSetCallback: public IUnknown
{
STDMETHOD(SetCallback)(IMatchFinderCallback *callback) PURE;
};
/*
// {23170F69-40C1-278A-0000-000200030000}
DEFINE_GUID(IID_IInitMatchFinder,
0x23170F69, 0x40C1, 0x278A, 0x00, 0x00, 0x00, 0x02, 0x00, 0x03, 0x00, 0x00);
MIDL_INTERFACE("23170F69-40C1-278A-0000-000200030000")
IMatchFinderInit: public IUnknown
{
STDMETHOD(InitMatchFinder)(IMatchFinder *matchFinder) PURE;
};
*/
#endif

19
7zip/Compress/LZ/LZInWindow.cpp
View File

@@ -16,7 +16,6 @@ bool CLZInWindow::Create(UInt32 keepSizeBefore, UInt32 keepSizeAfter, UInt32 kee
{
_keepSizeBefore = keepSizeBefore;
_keepSizeAfter = keepSizeAfter;
_keepSizeReserv = keepSizeReserv;
UInt32 blockSize = keepSizeBefore + keepSizeAfter + keepSizeReserv;
if (_bufferBase == 0 || _blockSize != blockSize)
{
@@ -31,10 +30,13 @@ bool CLZInWindow::Create(UInt32 keepSizeBefore, UInt32 keepSizeAfter, UInt32 kee
return (_bufferBase != 0);
}
HRESULT CLZInWindow::Init(ISequentialInStream *stream)
void CLZInWindow::SetStream(ISequentialInStream *stream)
{
_stream = stream;
}
HRESULT CLZInWindow::Init()
{
_buffer = _bufferBase;
_pos = 0;
_streamPos = 0;
@@ -68,7 +70,7 @@ HRESULT CLZInWindow::ReadBlock()
return S_OK;
while(true)
{
UInt32 size = UInt32(_bufferBase - _buffer) + _blockSize - _streamPos;
UInt32 size = (UInt32)(_bufferBase - _buffer) + _blockSize - _streamPos;
if(size == 0)
return S_OK;
UInt32 numReadBytes;
@@ -93,10 +95,11 @@ HRESULT CLZInWindow::ReadBlock()
void CLZInWindow::MoveBlock()
{
BeforeMoveBlock();
UInt32 offset = UInt32(_buffer - _bufferBase) + _pos - _keepSizeBefore;
UInt32 numBytes = UInt32(_buffer - _bufferBase) + _streamPos - offset;
UInt32 offset = (UInt32)(_buffer - _bufferBase) + _pos - _keepSizeBefore;
// we need one additional byte, since MovePos moves on 1 byte.
if (offset > 0)
offset--;
UInt32 numBytes = (UInt32)(_buffer - _bufferBase) + _streamPos - offset;
memmove(_bufferBase, _bufferBase + offset, numBytes);
_buffer -= offset;
AfterMoveBlock();
}

25
7zip/Compress/LZ/LZInWindow.h
View File

@@ -9,7 +9,7 @@ class CLZInWindow
{
Byte *_bufferBase; // pointer to buffer with data
ISequentialInStream *_stream;
UInt32 _posLimit; // offset (from _buffer) of first byte when new block reading must be done
UInt32 _posLimit; // offset (from _buffer) when new block reading must be done
bool _streamEndWasReached; // if (true) then _streamPos shows real end of stream
const Byte *_pointerToLastSafePosition;
protected:
@@ -18,22 +18,20 @@ protected:
UInt32 _pos; // offset (from _buffer) of curent byte
UInt32 _keepSizeBefore; // how many BYTEs must be kept in buffer before _pos
UInt32 _keepSizeAfter; // how many BYTEs must be kept buffer after _pos
UInt32 _keepSizeReserv; // how many BYTEs must be kept as reserv
UInt32 _streamPos; // offset (from _buffer) of first not read byte from Stream
virtual void BeforeMoveBlock() {};
virtual void AfterMoveBlock() {};
void MoveBlock();
virtual HRESULT ReadBlock();
HRESULT ReadBlock();
void Free();
public:
CLZInWindow(): _bufferBase(0) {}
virtual ~CLZInWindow() { Free(); }
bool Create(UInt32 keepSizeBefore, UInt32 keepSizeAfter,
UInt32 keepSizeReserv = (1<<17));
// keepSizeBefore + keepSizeAfter + keepSizeReserv < 4G)
bool Create(UInt32 keepSizeBefore, UInt32 keepSizeAfter, UInt32 keepSizeReserv = (1<<17));
HRESULT Init(ISequentialInStream *stream);
void SetStream(ISequentialInStream *stream);
HRESULT Init();
// void ReleaseStream();
Byte *GetBuffer() const { return _buffer; }
@@ -53,17 +51,17 @@ public:
else
return S_OK;
}
Byte GetIndexByte(Int32 index)const
{ return _buffer[(size_t)_pos + index]; }
Byte GetIndexByte(Int32 index) const { return _buffer[(size_t)_pos + index]; }
// index + limit have not to exceed _keepSizeAfter;
// -2G <= index < 2G
UInt32 GetMatchLen(Int32 index, UInt32 distance, UInt32 limit) const
{
if(_streamEndWasReached)
if ((_pos + index) + limit > _streamPos)
limit = _streamPos - (_pos + index);
distance++;
Byte *pby = _buffer + (size_t)_pos + index;
const Byte *pby = _buffer + (size_t)_pos + index;
UInt32 i;
for(i = 0; i < limit && pby[i] == pby[(size_t)i - distance]; i++);
return i;
@@ -79,6 +77,11 @@ public:
_streamPos -= subValue;
}
bool NeedMove(UInt32 numCheckBytes)
{
UInt32 reserv = _pointerToLastSafePosition - (_buffer + _pos);
return (reserv <= numCheckBytes);
}
};
#endif

463
7zip/Compress/LZ/MT/MT.cpp
View File

@@ -6,87 +6,96 @@
#include "MT.h"
class CMatchFinderCallback:
public IMatchFinderCallback,
public CMyUnknownImp
{
MY_UNKNOWN_IMP
static const UInt32 kBlockSize = (1 << 14);
STDMETHOD(BeforeChangingBufferPos)();
STDMETHOD(AfterChangingBufferPos)();
public:
CMatchFinderMT *m_MatchFinderMT;
const Byte *m_BufferPosBefore;
};
STDMETHODIMP CMatchFinderCallback::BeforeChangingBufferPos()
static DWORD WINAPI MFThread(void *threadCoderInfo)
{
m_MatchFinderMT->m_AskChangeBufferPos.Set();
m_MatchFinderMT->m_CanChangeBufferPos.Lock();
m_BufferPosBefore = m_MatchFinderMT->m_MatchFinder->GetPointerToCurrentPos();
return S_OK;
return ((CMatchFinderMT *)threadCoderInfo)->ThreadFunc();
}
STDMETHODIMP CMatchFinderCallback::AfterChangingBufferPos()
CMatchFinderMT::CMatchFinderMT():
m_Buffer(0),
m_NeedStart(true)
{
m_MatchFinderMT->m_DataCurrentPos +=
m_MatchFinderMT->m_MatchFinder->GetPointerToCurrentPos() - m_BufferPosBefore;
m_MatchFinderMT->m_BufferPosWasChanged.Set();
return S_OK;
m_BlockIndex = kNumMTBlocks - 1;
m_CS[m_BlockIndex].Enter();
if (!m_Thread.Create(MFThread, this))
throw 271826;
}
HRESULT CMatchFinderMT::SetMatchFinder(IMatchFinder *matchFinder,
UInt32 multiThreadMult)
CMatchFinderMT::~CMatchFinderMT()
{
_multiThreadMult = multiThreadMult;
m_MatchFinder = matchFinder;
CMyComPtr<IMatchFinderSetCallback> matchFinderSetCallback;
if (m_MatchFinder.QueryInterface(IID_IMatchFinderSetCallback,
&matchFinderSetCallback) == S_OK)
{
CMatchFinderCallback *matchFinderCallbackSpec =
new CMatchFinderCallback;
CMyComPtr<IMatchFinderCallback> matchFinderCallback = matchFinderCallbackSpec;
matchFinderCallbackSpec->m_MatchFinderMT = this;
matchFinderSetCallback->SetCallback(matchFinderCallback);
return S_OK;
}
else
return E_FAIL;
m_Exit = true;
m_CS[m_BlockIndex].Leave();
m_CanChangeBufferPos.Set();
if (m_NeedStart)
m_MtCanStart.Set();
m_Thread.Wait();
FreeMem();
}
void CMatchFinderMT::FreeMem()
{
::MyFree(m_Buffer);
m_Buffer = 0;
}
STDMETHODIMP CMatchFinderMT::Init(ISequentialInStream *s)
STDMETHODIMP CMatchFinderMT::Create(UInt32 sizeHistory, UInt32 keepAddBufferBefore,
UInt32 matchMaxLen, UInt32 keepAddBufferAfter)
{
m_AskChangeBufferPos.Reset();
m_CanChangeBufferPos.Reset();
m_BufferPosWasChanged.Reset();
m_StopWriting.Reset();
m_WritingWasStopped.Reset();
m_NeedStart = true;
m_CurrentPos = 0;
m_CurrentLimitPos = 0;
FreeMem();
m_MatchMaxLen = matchMaxLen;
if (kBlockSize <= matchMaxLen * 4)
return E_INVALIDARG;
UInt32 bufferSize = kBlockSize * kNumMTBlocks;
m_Buffer = (UInt32 *)::MyAlloc(bufferSize * sizeof(UInt32));
if (m_Buffer == 0)
return E_OUTOFMEMORY;
keepAddBufferBefore += bufferSize;
keepAddBufferAfter += (kBlockSize + 1);
return m_MatchFinder->Create(sizeHistory, keepAddBufferBefore, matchMaxLen, keepAddBufferAfter);
}
HRESULT result = m_MatchFinder->Init(s);
// UInt32 blockSizeMult = 800
HRESULT CMatchFinderMT::SetMatchFinder(IMatchFinder *matchFinder)
{
m_MatchFinder = matchFinder;
return S_OK;
}
STDMETHODIMP CMatchFinderMT::SetStream(ISequentialInStream *s)
{
return m_MatchFinder->SetStream(s);
}
// Call it after ReleaseStream / SetStream
STDMETHODIMP CMatchFinderMT::Init()
{
m_NeedStart = true;
m_Pos = 0;
m_PosLimit = 0;
HRESULT result = m_MatchFinder->Init();
if (result == S_OK)
m_DataCurrentPos = m_MatchFinder->GetPointerToCurrentPos();
m_NumAvailableBytes = m_MatchFinder->GetNumAvailableBytes();
return result;
}
// ReleaseStream is required to finish multithreading
STDMETHODIMP_(void) CMatchFinderMT::ReleaseStream()
{
m_StopWriting.Set();
m_WritingWasStopped.Lock();
m_MatchFinder->ReleaseStream();
}
STDMETHODIMP CMatchFinderMT::MovePos()
{
if (m_Result != S_OK)
return m_Result;
m_NumAvailableBytesCurrent--;
m_DataCurrentPos++;
return S_OK;
m_StopWriting = true;
m_CS[m_BlockIndex].Leave();
if (!m_NeedStart)
{
m_CanChangeBufferPos.Set();
m_MtWasStopped.Lock();
m_NeedStart = true;
}
m_MatchFinder->ReleaseStream();
m_BlockIndex = kNumMTBlocks - 1;
m_CS[m_BlockIndex].Enter();
}
STDMETHODIMP_(Byte) CMatchFinderMT::GetIndexByte(Int32 index)
@@ -94,11 +103,10 @@ STDMETHODIMP_(Byte) CMatchFinderMT::GetIndexByte(Int32 index)
return m_DataCurrentPos[index];
}
STDMETHODIMP_(UInt32) CMatchFinderMT::GetMatchLen(Int32 index,
UInt32 distance, UInt32 limit)
STDMETHODIMP_(UInt32) CMatchFinderMT::GetMatchLen(Int32 index, UInt32 distance, UInt32 limit)
{
if (int(index + limit) > m_NumAvailableBytesCurrent)
limit = m_NumAvailableBytesCurrent - (index);
if ((Int32)(index + limit) > m_NumAvailableBytes)
limit = m_NumAvailableBytes - (index);
distance++;
const Byte *pby = m_DataCurrentPos + index;
UInt32 i;
@@ -111,202 +119,177 @@ STDMETHODIMP_(const Byte *) CMatchFinderMT::GetPointerToCurrentPos()
return m_DataCurrentPos;
}
STDMETHODIMP_(UInt32) CMatchFinderMT::GetNumAvailableBytes()
{
if (m_NeedStart)
return m_MatchFinder->GetNumAvailableBytes();
else
return m_NumAvailableBytesCurrent;
return m_NumAvailableBytes;
}
void CMatchFinderMT::FreeMem()
void CMatchFinderMT::GetNextBlock()
{
MyFree(m_Buffer);
MyFree(m_DummyBuffer);
if (m_NeedStart)
{
m_NeedStart = false;
for (UInt32 i = 0; i < kNumMTBlocks; i++)
m_StopReading[i] = false;
m_StopWriting = false;
m_Exit = false;
m_MtWasStarted.Reset();
m_MtWasStopped.Reset();
m_CanChangeBufferPos.Reset();
m_BufferPosWasChanged.Reset();
m_MtCanStart.Set();
m_MtWasStarted.Lock();
m_Result = S_OK;
}
while (true)
{
UInt32 nextIndex = (m_BlockIndex == kNumMTBlocks - 1) ? 0 : m_BlockIndex + 1;
m_CS[nextIndex].Enter();
if (!m_StopReading[nextIndex])
{
m_CS[m_BlockIndex].Leave();
m_BlockIndex = nextIndex;
break;
}
m_StopReading[nextIndex] = false;
m_CS[nextIndex].Leave();
m_CanChangeBufferPos.Set();
m_BufferPosWasChanged.Lock();
m_CS[nextIndex].Enter();
m_CS[m_BlockIndex].Leave();
m_BlockIndex = nextIndex;
}
m_Pos = m_BlockIndex * kBlockSize;
m_PosLimit = m_Buffer[m_Pos++];
m_NumAvailableBytes = m_Buffer[m_Pos++];
m_Result = m_Results[m_BlockIndex];
}
STDMETHODIMP CMatchFinderMT::Create(UInt32 sizeHistory,
UInt32 keepAddBufferBefore, UInt32 matchMaxLen,
UInt32 keepAddBufferAfter)
STDMETHODIMP CMatchFinderMT::GetMatches(UInt32 *distances)
{
FreeMem();
m_MatchMaxLen = matchMaxLen;
if (m_Pos == m_PosLimit)
GetNextBlock();
m_BlockSize = (matchMaxLen + 1) * _multiThreadMult;
UInt32 bufferSize = m_BlockSize * kNumMTBlocks;
m_DummyBuffer = (UInt32 *)MyAlloc((matchMaxLen + 1) * sizeof(UInt32));
if (m_DummyBuffer == 0)
return E_OUTOFMEMORY;
m_Buffer = (UInt32 *)MyAlloc(bufferSize * sizeof(UInt32));
if (m_Buffer == 0)
return E_OUTOFMEMORY;
for (int i = 0; i < kNumMTBlocks; i++)
m_Buffers[i] = &m_Buffer[i * m_BlockSize];
if (m_Result != S_OK)
return m_Result;
m_NumAvailableBytes--;
m_DataCurrentPos++;
m_NeedStart = true;
m_CurrentPos = 0;
m_CurrentLimitPos = 0;
keepAddBufferBefore += bufferSize;
return m_MatchFinder->Create(sizeHistory, keepAddBufferBefore, matchMaxLen,
keepAddBufferAfter);
const UInt32 *buffer = m_Buffer + m_Pos;
UInt32 len = *buffer++;
*distances++ = len;
m_Pos += 1 + len;
for (UInt32 i = 0; i != len; i += 2)
{
distances[i] = buffer[i];
distances[i + 1] = buffer[i + 1];
}
return S_OK;
}
static DWORD WINAPI MFThread(void *threadCoderInfo)
STDMETHODIMP CMatchFinderMT::Skip(UInt32 num)
{
do
{
if (m_Pos == m_PosLimit)
GetNextBlock();
if (m_Result != S_OK)
return m_Result;
m_NumAvailableBytes--;
m_DataCurrentPos++;
UInt32 len = m_Buffer[m_Pos++];
m_Pos += len;
}
while(--num != 0);
return S_OK;
}
STDMETHODIMP_(Int32) CMatchFinderMT::NeedChangeBufferPos(UInt32 numCheckBytes)
{
CMatchFinderMT &mt = *(CMatchFinderMT *)threadCoderInfo;
return mt.ThreadFunc();
throw 1;
}
STDMETHODIMP_(void) CMatchFinderMT::ChangeBufferPos()
{
throw 1;
}
DWORD CMatchFinderMT::ThreadFunc()
{
bool errorMode = false;
while (true)
while(true)
{
HANDLE events[3] = { m_ExitEvent, m_StopWriting, m_CanWriteEvents[m_WriteBufferIndex] } ;
DWORD waitResult = ::WaitForMultipleObjects((errorMode ? 2: 3), events, FALSE, INFINITE);
if (waitResult == WAIT_OBJECT_0 + 0)
return 0;
if (waitResult == WAIT_OBJECT_0 + 1)
{
m_WriteBufferIndex = 0;
for (int i = 0; i < kNumMTBlocks; i++)
m_CanWriteEvents[i].Reset();
m_WritingWasStopped.Set();
errorMode = false;
continue;
}
if (errorMode)
{
// this case means bug_in_program. So just exit;
return 1;
}
m_Results[m_WriteBufferIndex] = S_OK;
UInt32 *buffer = m_Buffers[m_WriteBufferIndex];
UInt32 curPos = 0;
UInt32 numBytes = 0;
UInt32 limit = m_BlockSize - m_MatchMaxLen;
IMatchFinder *mf = m_MatchFinder;
do
{
if (mf->GetNumAvailableBytes() == 0)
break;
UInt32 len = mf->GetLongestMatch(buffer + curPos);
/*
if (len == 1)
len = 0;
*/
buffer[curPos] = len;
curPos += len + 1;
numBytes++;
HRESULT result = mf->MovePos();
if (result != S_OK)
{
m_Results[m_WriteBufferIndex] = result;
errorMode = true;
break;
}
}
while (curPos < limit);
m_LimitPos[m_WriteBufferIndex] = curPos;
if (errorMode)
m_NumAvailableBytes[m_WriteBufferIndex] = numBytes;
else
m_NumAvailableBytes[m_WriteBufferIndex] = numBytes +
mf->GetNumAvailableBytes();
m_CanReadEvents[m_WriteBufferIndex].Set();
if (++m_WriteBufferIndex == kNumMTBlocks)
m_WriteBufferIndex = 0;
}
}
CMatchFinderMT::CMatchFinderMT():
m_Buffer(0),
m_DummyBuffer(0),
_multiThreadMult(100)
{
for (int i = 0; i < kNumMTBlocks; i++)
{
m_CanReadEvents[i].Reset();
m_CanWriteEvents[i].Reset();
}
m_ReadBufferIndex = 0;
m_WriteBufferIndex = 0;
m_ExitEvent.Reset();
if (!m_Thread.Create(MFThread, this))
throw 271826;
}
CMatchFinderMT::~CMatchFinderMT()
{
m_ExitEvent.Set();
m_Thread.Wait();
FreeMem();
}
void CMatchFinderMT::Start()
{
m_AskChangeBufferPos.Reset();
m_CanChangeBufferPos.Reset();
m_BufferPosWasChanged.Reset();
m_WriteBufferIndex = 0;
m_ReadBufferIndex = 0;
m_NeedStart = false;
m_CurrentPos = 0;
m_CurrentLimitPos = 0;
m_Result = S_OK;
int i;
for (i = 0; i < kNumMTBlocks; i++)
m_CanReadEvents[i].Reset();
for (i = kNumMTBlocks - 1; i >= 0; i--)
m_CanWriteEvents[i].Set();
}
STDMETHODIMP_(UInt32) CMatchFinderMT::GetLongestMatch(UInt32 *distances)
{
if (m_CurrentPos == m_CurrentLimitPos)
{
if (m_NeedStart)
Start();
bool needStartEvent = true;
m_MtCanStart.Lock();
HRESULT result = S_OK;
UInt32 blockIndex = 0;
while (true)
{
HANDLE events[2] = { m_AskChangeBufferPos, m_CanReadEvents[m_ReadBufferIndex] } ;
DWORD waitResult = ::WaitForMultipleObjects(2, events, FALSE, INFINITE);
if (waitResult == WAIT_OBJECT_0 + 1)
m_CS[blockIndex].Enter();
if (needStartEvent)
{
m_MtWasStarted.Set();
needStartEvent = false;
}
else
m_CS[(blockIndex == 0) ? kNumMTBlocks - 1 : blockIndex - 1].Leave();
if (m_Exit)
return 0;
if (m_StopWriting)
{
m_MtWasStopped.Set();
m_CS[blockIndex].Leave();
break;
m_BufferPosWasChanged.Reset();
m_CanChangeBufferPos.Set();
m_BufferPosWasChanged.Lock();
}
if (result == S_OK)
{
IMatchFinder *mf = m_MatchFinder;
if (mf->NeedChangeBufferPos(kBlockSize) != 0)
{
// m_AskChangeBufferPos.Set();
m_StopReading[blockIndex] = true;
m_CS[blockIndex].Leave();
m_CanChangeBufferPos.Lock();
m_CS[blockIndex].Enter();
const Byte *bufferPosBefore = mf->GetPointerToCurrentPos();
mf->ChangeBufferPos();
m_DataCurrentPos += mf->GetPointerToCurrentPos() - bufferPosBefore;
m_BufferPosWasChanged.Set();
}
else
{
UInt32 curPos = blockIndex * kBlockSize;
UInt32 limit = curPos + kBlockSize - m_MatchMaxLen - m_MatchMaxLen - 1;
UInt32 *buffer = m_Buffer;
m_Results[blockIndex] = S_OK;
curPos++;
UInt32 numAvailableBytes = mf->GetNumAvailableBytes();
buffer[curPos++] = numAvailableBytes;
while (numAvailableBytes-- != 0 && curPos < limit)
{
result = mf->GetMatches(buffer + curPos);
if (result != S_OK)
{
m_Results[blockIndex] = result;
break;
}
curPos += buffer[curPos] + 1;
}
buffer[blockIndex * kBlockSize] = curPos;
}
}
else
{
UInt32 curPos = blockIndex * kBlockSize;
m_Buffer[curPos] = curPos + 2; // size of buffer
m_Buffer[curPos + 1] = 0; // NumAvailableBytes
m_Results[blockIndex] = result; // error
}
if (++blockIndex == kNumMTBlocks)
blockIndex = 0;
}
m_CurrentLimitPos = m_LimitPos[m_ReadBufferIndex];
m_NumAvailableBytesCurrent = m_NumAvailableBytes[m_ReadBufferIndex];
m_CurrentPos = 0;
m_Result = m_Results[m_ReadBufferIndex];
}
/*
if (m_CurrentPos >= m_CurrentLimitPos)
throw 1123324;
*/
const UInt32 *buffer = m_Buffers[m_ReadBufferIndex];
UInt32 len = buffer[m_CurrentPos++];
for (UInt32 i = 1; i <= len; i++)
distances[i] = buffer[m_CurrentPos++];
if (m_CurrentPos == m_CurrentLimitPos)
{
m_CanWriteEvents[m_ReadBufferIndex].Set();
if (++m_ReadBufferIndex == kNumMTBlocks)
m_ReadBufferIndex = 0;
}
return len;
}
STDMETHODIMP_(void) CMatchFinderMT::DummyLongestMatch()
{
GetLongestMatch(m_DummyBuffer);
}

74
7zip/Compress/LZ/MT/MT.h
View File

@@ -11,7 +11,7 @@
#include "../../../ICoder.h"
#include "../IMatchFinder.h"
const int kNumMTBlocks = 3;
const UInt32 kNumMTBlocks = (1 << 6);
class CMatchFinderMT:
public IMatchFinder,
@@ -19,65 +19,61 @@ class CMatchFinderMT:
{
MY_UNKNOWN_IMP
STDMETHOD(Init)(ISequentialInStream *s);
STDMETHOD(SetStream)(ISequentialInStream *inStream);
STDMETHOD_(void, ReleaseStream)();
STDMETHOD(MovePos)();
STDMETHOD(Init)();
STDMETHOD_(Byte, GetIndexByte)(Int32 index);
STDMETHOD_(UInt32, GetMatchLen)(Int32 index, UInt32 distance, UInt32 limit);
STDMETHOD_(UInt32, GetNumAvailableBytes)();
STDMETHOD_(const Byte *, GetPointerToCurrentPos)();
STDMETHOD(Create)(UInt32 sizeHistory,
UInt32 keepAddBufferBefore, UInt32 matchMaxLen,
UInt32 keepAddBufferAfter);
STDMETHOD_(UInt32, GetLongestMatch)(UInt32 *distances);
STDMETHOD_(void, DummyLongestMatch)();
STDMETHOD(Create)(UInt32 sizeHistory, UInt32 keepAddBufferBefore,
UInt32 matchMaxLen, UInt32 keepAddBufferAfter);
STDMETHOD(GetMatches)(UInt32 *distances);
STDMETHOD(Skip)(UInt32 num);
UInt32 m_CurrentPos;
UInt32 m_CurrentLimitPos;
STDMETHOD_(Int32, NeedChangeBufferPos)(UInt32 numCheckBytes);
STDMETHOD_(void, ChangeBufferPos)();
UInt32 m_Pos;
UInt32 m_PosLimit;
UInt32 m_MatchMaxLen;
UInt32 m_BlockSize;
UInt32 *m_Buffer;
UInt32 *m_Buffers[kNumMTBlocks];
UInt32 *m_DummyBuffer;
bool m_NeedStart;
UInt32 m_WriteBufferIndex;
UInt32 m_ReadBufferIndex;
NWindows::NSynchronization::CAutoResetEvent m_StopWriting;
NWindows::NSynchronization::CAutoResetEvent m_WritingWasStopped;
NWindows::NSynchronization::CManualResetEvent m_ExitEvent;
NWindows::NSynchronization::CAutoResetEvent m_CanReadEvents[kNumMTBlocks];
NWindows::NSynchronization::CAutoResetEvent m_CanWriteEvents[kNumMTBlocks];
HRESULT m_Results[kNumMTBlocks];
UInt32 m_LimitPos[kNumMTBlocks];
UInt32 m_NumAvailableBytes[kNumMTBlocks];
UInt32 m_NumAvailableBytesCurrent;
NWindows::CThread m_Thread;
UInt32 _multiThreadMult;
UInt32 m_BlockIndex;
HRESULT m_Result;
UInt32 m_NumAvailableBytes;
const Byte *m_DataCurrentPos;
void Start();
void FreeMem();
// Common variables
public:
NWindows::NSynchronization::CAutoResetEvent m_AskChangeBufferPos;
CMyComPtr<IMatchFinder> m_MatchFinder;
NWindows::CThread m_Thread;
NWindows::NSynchronization::CAutoResetEvent m_MtCanStart;
NWindows::NSynchronization::CAutoResetEvent m_MtWasStarted;
NWindows::NSynchronization::CAutoResetEvent m_MtWasStopped;
NWindows::NSynchronization::CAutoResetEvent m_CanChangeBufferPos;
NWindows::NSynchronization::CAutoResetEvent m_BufferPosWasChanged;
CMyComPtr<IMatchFinder> m_MatchFinder;
const Byte *m_DataCurrentPos;
NWindows::NSynchronization::CCriticalSection m_CS[kNumMTBlocks];
HRESULT m_Results[kNumMTBlocks];
bool m_StopReading[kNumMTBlocks];
bool m_Exit;
bool m_StopWriting;
////////////////////////////
void FreeMem();
void GetNextBlock();
public:
DWORD ThreadFunc();
CMatchFinderMT();
~CMatchFinderMT();
HRESULT SetMatchFinder(IMatchFinder *matchFinder, UInt32 multiThreadMult = 200);
HRESULT SetMatchFinder(IMatchFinder *matchFinder);
};
#endif

318
7zip/Compress/LZ/Patricia/Pat.h
View File

@@ -1,318 +0,0 @@
// Pat.h
// #ifndef __PATRICIA__H
// #define __PATRICIA__H
#include "../../../../Common/MyCom.h"
#include "../../../../Common/Types.h"
#include "../LZInWindow.h"
namespace PAT_NAMESPACE {
struct CNode;
typedef CNode *CNodePointer;
// #define __AUTO_REMOVE
// #define __NODE_4_BITS
// #define __NODE_3_BITS
// #define __NODE_2_BITS
// #define __NODE_2_BITS_PADDING
// #define __HASH_3
typedef UInt32 CIndex;
#ifdef __NODE_4_BITS
typedef UInt32 CIndex2;
typedef UInt32 CSameBitsType;
#else
#ifdef __NODE_3_BITS
typedef UInt32 CIndex2;
typedef UInt32 CSameBitsType;
#else
typedef UInt32 CIndex;
typedef UInt32 CSameBitsType;
typedef CIndex CIndex2;
#endif
#endif
const UInt32 kNumBitsInIndex = sizeof(CIndex) * 8;
const UInt32 kMatchStartValue = UInt32(1) << (kNumBitsInIndex - 1);
// don't change kMatchStartValue definition, since it is used in
// PatMain.h:
typedef CIndex CMatchPointer;
const UInt32 kDescendantEmptyValue = kMatchStartValue - 1;
union CDescendant
{
CIndex NodePointer;
CMatchPointer MatchPointer;
bool IsEmpty() const { return NodePointer == kDescendantEmptyValue; }
bool IsNode() const { return NodePointer < kDescendantEmptyValue; }
bool IsMatch() const { return NodePointer > kDescendantEmptyValue; }
void MakeEmpty() { NodePointer = kDescendantEmptyValue; }
};
#undef MY_BYTE_SIZE
#ifdef __NODE_4_BITS
#define MY_BYTE_SIZE 8
const UInt32 kNumSubBits = 4;
#else
#ifdef __NODE_3_BITS
#define MY_BYTE_SIZE 9
const UInt32 kNumSubBits = 3;
#else
#define MY_BYTE_SIZE 8
#ifdef __NODE_2_BITS
const UInt32 kNumSubBits = 2;
#else
const UInt32 kNumSubBits = 1;
#endif
#endif
#endif
const UInt32 kNumSubNodes = 1 << kNumSubBits;
const UInt32 kSubNodesMask = kNumSubNodes - 1;
struct CNode
{
CIndex2 LastMatch;
CSameBitsType NumSameBits;
union
{
CDescendant Descendants[kNumSubNodes];
UInt32 NextFreeNode;
};
#ifdef __NODE_2_BITS
#ifdef __NODE_2_BITS_PADDING
UInt32 Padding[2];
#endif
#endif
};
#undef kIDNumBitsByte
#undef kIDNumBitsString
#ifdef __NODE_4_BITS
#define kIDNumBitsByte 0x30
#define kIDNumBitsString TEXT("4")
#else
#ifdef __NODE_3_BITS
#define kIDNumBitsByte 0x20
#define kIDNumBitsString TEXT("3")
#else
#ifdef __NODE_2_BITS
#define kIDNumBitsByte 0x10
#define kIDNumBitsString TEXT("2")
#else
#define kIDNumBitsByte 0x00
#define kIDNumBitsString TEXT("1")
#endif
#endif
#endif
#undef kIDManualRemoveByte
#undef kIDManualRemoveString
#ifdef __AUTO_REMOVE
#define kIDManualRemoveByte 0x00
#define kIDManualRemoveString TEXT("")
#else
#define kIDManualRemoveByte 0x08
#define kIDManualRemoveString TEXT("R")
#endif
#undef kIDHash3Byte
#undef kIDHash3String
#ifdef __HASH_3
#define kIDHash3Byte 0x04
#define kIDHash3String TEXT("H")
#else
#define kIDHash3Byte 0x00
#define kIDHash3String TEXT("")
#endif
#undef kIDUse3BytesByte
#undef kIDUse3BytesString
#define kIDUse3BytesByte 0x00
#define kIDUse3BytesString TEXT("")
#undef kIDPaddingByte
#undef kIDPaddingString
#ifdef __NODE_2_BITS_PADDING
#define kIDPaddingByte 0x01
#define kIDPaddingString TEXT("P")
#else
#define kIDPaddingByte 0x00
#define kIDPaddingString TEXT("")
#endif
// #undef kIDString
// #define kIDString TEXT("Compress.MatchFinderPat") kIDNumBitsString kIDManualRemoveString kIDUse3BytesString kIDPaddingString kIDHash3String
// {23170F69-40C1-278C-01XX-0000000000}
DEFINE_GUID(PAT_CLSID,
0x23170F69, 0x40C1, 0x278C, 0x01,
kIDNumBitsByte |
kIDManualRemoveByte | kIDHash3Byte | kIDUse3BytesByte | kIDPaddingByte,
0x00, 0x00, 0x00, 0x00, 0x00, 0x00);
// III(PAT_NAMESPACE)
class CPatricia:
public IMatchFinder,
public IMatchFinderSetCallback,
public CMyUnknownImp,
CLZInWindow
{
MY_UNKNOWN_IMP1(IMatchFinderSetCallback)
STDMETHOD(Init)(ISequentialInStream *aStream);
STDMETHOD_(void, ReleaseStream)();
STDMETHOD(MovePos)();
STDMETHOD_(Byte, GetIndexByte)(Int32 index);
STDMETHOD_(UInt32, GetMatchLen)(Int32 index, UInt32 back, UInt32 limit);
STDMETHOD_(UInt32, GetNumAvailableBytes)();
STDMETHOD(Create)(UInt32 historySize,
UInt32 keepAddBufferBefore, UInt32 matchMaxLen,
UInt32 keepAddBufferAfter);
STDMETHOD_(UInt32, GetLongestMatch)(UInt32 *distances);
STDMETHOD_(void, DummyLongestMatch)();
STDMETHOD_(const Byte *, GetPointerToCurrentPos)();
void FreeMemory();
public:
CPatricia();
~CPatricia();
UInt32 _sizeHistory;
UInt32 _matchMaxLen;
CDescendant *m_HashDescendants;
#ifdef __HASH_3
CDescendant *m_Hash2Descendants;
#endif
CNode *m_Nodes;
UInt32 m_FreeNode;
UInt32 m_FreeNodeMax;
#ifdef __AUTO_REMOVE
UInt32 m_NumUsedNodes;
UInt32 m_NumNodes;
#else
bool m_SpecialRemoveMode;
#endif
bool m_SpecialMode;
UInt32 m_NumNotChangedCycles;
UInt32 *m_TmpBacks;
CMyComPtr<IMatchFinderCallback> m_Callback;
virtual void BeforeMoveBlock();
virtual void AfterMoveBlock();
// IMatchFinderSetCallback
STDMETHOD(SetCallback)(IMatchFinderCallback *callback);
void ChangeLastMatch(UInt32 hashValue);
#ifdef __AUTO_REMOVE
void TestRemoveDescendant(CDescendant &descendant, UInt32 limitPos);
void TestRemoveNodes();
void RemoveNode(UInt32 index);
void TestRemoveAndNormalizeDescendant(CDescendant &descendant,
UInt32 limitPos, UInt32 subValue);
void TestRemoveNodesAndNormalize();
#else
void NormalizeDescendant(CDescendant &descendant, UInt32 subValue);
void Normalize();
void RemoveMatch();
#endif
private:
void AddInternalNode(CNodePointer aNode, CIndex *aNodePointerPointer,
Byte aByte, Byte aByteXOR, UInt32 aNumSameBits, UInt32 aPos)
{
while((aByteXOR & kSubNodesMask) == 0)
{
aByteXOR >>= kNumSubBits;
aByte >>= kNumSubBits;
aNumSameBits -= kNumSubBits;
}
// Insert New Node
CNodePointer aNewNode = &m_Nodes[m_FreeNode];
UInt32 aNodeIndex = *aNodePointerPointer;
*aNodePointerPointer = m_FreeNode;
m_FreeNode = aNewNode->NextFreeNode;
#ifdef __AUTO_REMOVE
m_NumUsedNodes++;
#endif
if (m_FreeNode > m_FreeNodeMax)
{
m_FreeNodeMax = m_FreeNode;
m_Nodes[m_FreeNode].NextFreeNode = m_FreeNode + 1;
}
UInt32 aBitsNew = aByte & kSubNodesMask;
UInt32 aBitsOld = (aByte ^ aByteXOR) & kSubNodesMask;
for (UInt32 i = 0; i < kNumSubNodes; i++)
aNewNode->Descendants[i].NodePointer = kDescendantEmptyValue;
aNewNode->Descendants[aBitsNew].MatchPointer = aPos + kMatchStartValue;
aNewNode->Descendants[aBitsOld].NodePointer = aNodeIndex;
aNewNode->NumSameBits = CSameBitsType(aNode->NumSameBits - aNumSameBits);
aNewNode->LastMatch = aPos;
aNode->NumSameBits = CSameBitsType(aNumSameBits - kNumSubBits);
}
void AddLeafNode(CNodePointer aNode, Byte aByte, Byte aByteXOR,
UInt32 aNumSameBits, UInt32 aPos, UInt32 aDescendantIndex)
{
for(;(aByteXOR & kSubNodesMask) == 0; aNumSameBits += kNumSubBits)
{
aByte >>= kNumSubBits;
aByteXOR >>= kNumSubBits;
}
UInt32 aNewNodeIndex = m_FreeNode;
CNodePointer aNewNode = &m_Nodes[m_FreeNode];
m_FreeNode = aNewNode->NextFreeNode;
#ifdef __AUTO_REMOVE
m_NumUsedNodes++;
#endif
if (m_FreeNode > m_FreeNodeMax)
{
m_FreeNodeMax = m_FreeNode;
m_Nodes[m_FreeNode].NextFreeNode = m_FreeNode + 1;
}
UInt32 aBitsNew = (aByte & kSubNodesMask);
UInt32 aBitsOld = (aByte ^ aByteXOR) & kSubNodesMask;
for (UInt32 i = 0; i < kNumSubNodes; i++)
aNewNode->Descendants[i].NodePointer = kDescendantEmptyValue;
aNewNode->Descendants[aBitsNew].MatchPointer = aPos + kMatchStartValue;
aNewNode->Descendants[aBitsOld].MatchPointer =
aNode->Descendants[aDescendantIndex].MatchPointer;
aNewNode->NumSameBits = CSameBitsType(aNumSameBits);
aNewNode->LastMatch = aPos;
aNode->Descendants[aDescendantIndex].NodePointer = aNewNodeIndex;
}
};
}
// #endif

22
7zip/Compress/LZ/Patricia/Pat2.h
View File

@@ -1,22 +0,0 @@
// Pat2.h
#ifndef __PAT2__H
#define __PAT2__H
#undef PAT_CLSID
#define PAT_CLSID CLSID_CMatchFinderPat2
#undef PAT_NAMESPACE
#define PAT_NAMESPACE NPat2
#define __AUTO_REMOVE
#define __NODE_2_BITS
#include "Pat.h"
#include "PatMain.h"
#undef __AUTO_REMOVE
#undef __NODE_2_BITS
#endif

24
7zip/Compress/LZ/Patricia/Pat2H.h
View File

@@ -1,24 +0,0 @@
// Pat2H.h
#ifndef __PAT2H__H
#define __PAT2H__H
#undef PAT_CLSID
#define PAT_CLSID CLSID_CMatchFinderPat2H
#undef PAT_NAMESPACE
#define PAT_NAMESPACE NPat2H
#define __AUTO_REMOVE
#define __NODE_2_BITS
#define __HASH_3
#include "Pat.h"
#include "PatMain.h"
#undef __AUTO_REMOVE
#undef __NODE_2_BITS
#undef __HASH_3
#endif

20
7zip/Compress/LZ/Patricia/Pat2R.h
View File

@@ -1,20 +0,0 @@
// Pat2R.h
#ifndef __PAT2R__H
#define __PAT2R__H
#undef PAT_CLSID
#define PAT_CLSID CLSID_CMatchFinderPat2R
#undef PAT_NAMESPACE
#define PAT_NAMESPACE NPat2R
#define __NODE_2_BITS
#include "Pat.h"
#include "PatMain.h"
#undef __NODE_2_BITS
#endif

24
7zip/Compress/LZ/Patricia/Pat3H.h
View File

@@ -1,24 +0,0 @@
// Pat3H.h
#ifndef __PAT3H__H
#define __PAT3H__H
#undef PAT_CLSID
#define PAT_CLSID CLSID_CMatchFinderPat3H
#undef PAT_NAMESPACE
#define PAT_NAMESPACE NPat3H
#define __AUTO_REMOVE
#define __NODE_3_BITS
#define __HASH_3
#include "Pat.h"
#include "PatMain.h"
#undef __AUTO_REMOVE
#undef __NODE_3_BITS
#undef __HASH_3
#endif

24
7zip/Compress/LZ/Patricia/Pat4H.h
View File

@@ -1,24 +0,0 @@
// Pat4H.h
#ifndef __PAT4H__H
#define __PAT4H__H
#undef PAT_CLSID
#define PAT_CLSID CLSID_CMatchFinderPat4H
#undef PAT_NAMESPACE
#define PAT_NAMESPACE NPat4H
#define __AUTO_REMOVE
#define __NODE_4_BITS
#define __HASH_3
#include "Pat.h"
#include "PatMain.h"
#undef __AUTO_REMOVE
#undef __NODE_4_BITS
#undef __HASH_3
#endif

989
7zip/Compress/LZ/Patricia/PatMain.h
View File

@@ -1,989 +0,0 @@
// PatMain.h
#include "../../../../Common/Defs.h"
#include "../../../../Common/Alloc.h"
namespace PAT_NAMESPACE {
STDMETHODIMP CPatricia::SetCallback(IMatchFinderCallback *callback)
{
m_Callback = callback;
return S_OK;
}
void CPatricia::BeforeMoveBlock()
{
if (m_Callback)
m_Callback->BeforeChangingBufferPos();
CLZInWindow::BeforeMoveBlock();
}
void CPatricia::AfterMoveBlock()
{
if (m_Callback)
m_Callback->AfterChangingBufferPos();
CLZInWindow::AfterMoveBlock();
}
const UInt32 kMatchStartValue2 = 2;
const UInt32 kDescendantEmptyValue2 = kMatchStartValue2 - 1;
const UInt32 kDescendantsNotInitilized2 = kDescendantEmptyValue2 - 1;
#ifdef __HASH_3
static const UInt32 kNumHashBytes = 3;
static const UInt32 kHashSize = 1 << (8 * kNumHashBytes);
static const UInt32 kNumHash2Bytes = 2;
static const UInt32 kHash2Size = 1 << (8 * kNumHash2Bytes);
static const UInt32 kPrevHashSize = kNumHash2Bytes;
#else
static const UInt32 kNumHashBytes = 2;
static const UInt32 kHashSize = 1 << (8 * kNumHashBytes);
static const UInt32 kPrevHashSize = 0;
#endif
CPatricia::CPatricia():
m_HashDescendants(0),
#ifdef __HASH_3
m_Hash2Descendants(0),
#endif
m_Nodes(0),
m_TmpBacks(0)
{
}
CPatricia::~CPatricia()
{
FreeMemory();
}
void CPatricia::FreeMemory()
{
MyFree(m_TmpBacks);
m_TmpBacks = 0;
::BigFree(m_Nodes);
m_Nodes = 0;
::BigFree(m_HashDescendants);
m_HashDescendants = 0;
#ifdef __HASH_3
::BigFree(m_Hash2Descendants);
m_Hash2Descendants = 0;
CLZInWindow::Free();
#endif
}
STDMETHODIMP CPatricia::Create(UInt32 historySize, UInt32 keepAddBufferBefore,
UInt32 matchMaxLen, UInt32 keepAddBufferAfter)
{
FreeMemory();
int kNumBitsInNumSameBits = sizeof(CSameBitsType) * 8;
if (kNumBitsInNumSameBits < 32 && ((matchMaxLen * MY_BYTE_SIZE) > ((UInt32)1 << kNumBitsInNumSameBits)))
return E_INVALIDARG;
const UInt32 kAlignMask = (1 << 16) - 1;
UInt32 windowReservSize = historySize;
windowReservSize += kAlignMask;
windowReservSize &= ~(kAlignMask);
const UInt32 kMinReservSize = (1 << 19);
if (windowReservSize < kMinReservSize)
windowReservSize = kMinReservSize;
windowReservSize += 256;
if (!CLZInWindow::Create(historySize + keepAddBufferBefore,
matchMaxLen + keepAddBufferAfter, windowReservSize))
return E_OUTOFMEMORY;
_sizeHistory = historySize;
_matchMaxLen = matchMaxLen;
m_HashDescendants = (CDescendant *)BigAlloc(kHashSize * sizeof(CDescendant));
if (m_HashDescendants == 0)
{
FreeMemory();
return E_OUTOFMEMORY;
}
#ifdef __HASH_3
m_Hash2Descendants = (CDescendant *)BigAlloc(kHash2Size * sizeof(CDescendant));
if (m_Hash2Descendants == 0)
{
FreeMemory();
return E_OUTOFMEMORY;
}
#endif
#ifdef __AUTO_REMOVE
#ifdef __HASH_3
m_NumNodes = historySize + _sizeHistory * 4 / 8 + (1 << 19);
#else
m_NumNodes = historySize + _sizeHistory * 4 / 8 + (1 << 10);
#endif
#else
UInt32 m_NumNodes = historySize;
#endif
const UInt32 kMaxNumNodes = UInt32(1) << (sizeof(CIndex) * 8 - 1);
if (m_NumNodes + 32 > kMaxNumNodes)
return E_INVALIDARG;
// m_Nodes = (CNode *)::BigAlloc((m_NumNodes + 2) * sizeof(CNode));
m_Nodes = (CNode *)::BigAlloc((m_NumNodes + 12) * sizeof(CNode));
if (m_Nodes == 0)
{
FreeMemory();
return E_OUTOFMEMORY;
}
m_TmpBacks = (UInt32 *)MyAlloc((_matchMaxLen + 1) * sizeof(UInt32));
if (m_TmpBacks == 0)
{
FreeMemory();
return E_OUTOFMEMORY;
}
return S_OK;
}
STDMETHODIMP CPatricia::Init(ISequentialInStream *aStream)
{
RINOK(CLZInWindow::Init(aStream));
// memset(m_HashDescendants, 0xFF, kHashSize * sizeof(m_HashDescendants[0]));
#ifdef __HASH_3
for (UInt32 i = 0; i < kHash2Size; i++)
m_Hash2Descendants[i].MatchPointer = kDescendantsNotInitilized2;
#else
for (UInt32 i = 0; i < kHashSize; i++)
m_HashDescendants[i].MakeEmpty();
#endif
m_Nodes[0].NextFreeNode = 1;
m_FreeNode = 0;
m_FreeNodeMax = 0;
#ifdef __AUTO_REMOVE
m_NumUsedNodes = 0;
#else
m_SpecialRemoveMode = false;
#endif
m_SpecialMode = false;
return S_OK;
}
STDMETHODIMP_(void) CPatricia::ReleaseStream()
{
// CLZInWindow::ReleaseStream();
}
// pos = _pos + kNumHashBytes
// fullCurrentLimit = currentLimit + kNumHashBytes
// fullMatchLen = matchLen + kNumHashBytes
void CPatricia::ChangeLastMatch(UInt32 hashValue)
{
UInt32 pos = _pos + kNumHashBytes - 1;
UInt32 descendantIndex;
const Byte *currentBytePointer = _buffer + pos;
UInt32 numLoadedBits = 0;
Byte curByte = 0; // = 0 to disable warning of GCC
CNodePointer node = &m_Nodes[m_HashDescendants[hashValue].NodePointer];
while(true)
{
UInt32 numSameBits = node->NumSameBits;
if(numSameBits > 0)
{
if (numLoadedBits < numSameBits)
{
numSameBits -= numLoadedBits;
currentBytePointer += (numSameBits / MY_BYTE_SIZE);
numSameBits %= MY_BYTE_SIZE;
curByte = *currentBytePointer++;
numLoadedBits = MY_BYTE_SIZE;
}
curByte >>= numSameBits;
numLoadedBits -= numSameBits;
}
if(numLoadedBits == 0)
{
curByte = *currentBytePointer++;
numLoadedBits = MY_BYTE_SIZE;
}
descendantIndex = (curByte & kSubNodesMask);
node->LastMatch = pos;
numLoadedBits -= kNumSubBits;
curByte >>= kNumSubBits;
if(node->Descendants[descendantIndex].IsNode())
node = &m_Nodes[node->Descendants[descendantIndex].NodePointer];
else
break;
}
node->Descendants[descendantIndex].MatchPointer = pos + kMatchStartValue;
}
UInt32 CPatricia::GetLongestMatch(UInt32 *distances)
{
UInt32 fullCurrentLimit;
if (_pos + _matchMaxLen <= _streamPos)
fullCurrentLimit = _matchMaxLen;
else
{
fullCurrentLimit = _streamPos - _pos;
if(fullCurrentLimit < kNumHashBytes)
return 0;
}
UInt32 pos = _pos + kNumHashBytes;
#ifdef __HASH_3
UInt32 hash2Value = ((UInt32(_buffer[_pos])) << 8) | _buffer[_pos + 1];
UInt32 hashValue = (hash2Value << 8) | _buffer[_pos + 2];
CDescendant &hash2Descendant = m_Hash2Descendants[hash2Value];
CDescendant &hashDescendant = m_HashDescendants[hashValue];
if(hash2Descendant.MatchPointer <= kDescendantEmptyValue2)
{
if(hash2Descendant.MatchPointer == kDescendantsNotInitilized2)
{
UInt32 base = hashValue & 0xFFFF00;
for (UInt32 i = 0; i < 0x100; i++)
m_HashDescendants[base + i].MakeEmpty();
}
hash2Descendant.MatchPointer = pos + kMatchStartValue2;
hashDescendant.MatchPointer = pos + kMatchStartValue;
return 0;
}
distances[kNumHash2Bytes] = pos - (hash2Descendant.MatchPointer - kMatchStartValue2) - 1;
hash2Descendant.MatchPointer = pos + kMatchStartValue2;
#ifdef __AUTO_REMOVE
if (distances[kNumHash2Bytes] >= _sizeHistory)
{
if (hashDescendant.IsNode())
RemoveNode(hashDescendant.NodePointer);
hashDescendant.MatchPointer = pos + kMatchStartValue;
return 0;
}
#endif
if (fullCurrentLimit == kNumHash2Bytes)
return kNumHash2Bytes;
#else
UInt32 hashValue = UInt32(GetIndexByte(1)) | (UInt32(GetIndexByte(0)) << 8);
CDescendant &hashDescendant = m_HashDescendants[hashValue];
#endif
if(m_SpecialMode)
{
if(hashDescendant.IsMatch())
m_NumNotChangedCycles = 0;
if(m_NumNotChangedCycles >= _sizeHistory - 1)
{
ChangeLastMatch(hashValue);
m_NumNotChangedCycles = 0;
}
if(GetIndexByte(fullCurrentLimit - 1) == GetIndexByte(fullCurrentLimit - 2))
{
if(hashDescendant.IsMatch())
hashDescendant.MatchPointer = pos + kMatchStartValue;
else
m_NumNotChangedCycles++;
for(UInt32 i = kNumHashBytes; i <= fullCurrentLimit; i++)
distances[i] = 0;
return fullCurrentLimit;
}
else if(m_NumNotChangedCycles > 0)
ChangeLastMatch(hashValue);
m_SpecialMode = false;
}
if(hashDescendant.IsEmpty())
{
hashDescendant.MatchPointer = pos + kMatchStartValue;
return kPrevHashSize;
}
UInt32 currentLimit = fullCurrentLimit - kNumHashBytes;
if(hashDescendant.IsMatch())
{
CMatchPointer matchPointer = hashDescendant.MatchPointer;
UInt32 backReal = pos - (matchPointer - kMatchStartValue);
UInt32 back = backReal - 1;
#ifdef __AUTO_REMOVE
if (back >= _sizeHistory)
{
hashDescendant.MatchPointer = pos + kMatchStartValue;
return kPrevHashSize;
}
#endif
UInt32 matchLen;
distances += kNumHashBytes;
Byte *buffer = _buffer + pos;
for(matchLen = 0; true; matchLen++)
{
*distances++ = back;
if (matchLen == currentLimit)
{
hashDescendant.MatchPointer = pos + kMatchStartValue;
return kNumHashBytes + matchLen;
}
if (buffer[matchLen] != buffer[(size_t)matchLen - backReal])
break;
}
// UInt32 matchLen = GetMatchLen(kNumHashBytes, back, currentLimit);
UInt32 fullMatchLen = matchLen + kNumHashBytes;
hashDescendant.NodePointer = m_FreeNode;
CNodePointer node = &m_Nodes[m_FreeNode];
m_FreeNode = node->NextFreeNode;
#ifdef __AUTO_REMOVE
m_NumUsedNodes++;
#endif
if (m_FreeNode > m_FreeNodeMax)
{
m_FreeNodeMax = m_FreeNode;
m_Nodes[m_FreeNode].NextFreeNode = m_FreeNode + 1;
}
for (UInt32 i = 0; i < kNumSubNodes; i++)
node->Descendants[i].NodePointer = kDescendantEmptyValue;
node->LastMatch = pos;
Byte byteNew = GetIndexByte(fullMatchLen);
Byte byteOld = GetIndexByte(fullMatchLen - backReal);
Byte bitsNew, bitsOld;
UInt32 numSameBits = matchLen * MY_BYTE_SIZE;
while (true)
{
bitsNew = (byteNew & kSubNodesMask);
bitsOld = (byteOld & kSubNodesMask);
if(bitsNew != bitsOld)
break;
byteNew >>= kNumSubBits;
byteOld >>= kNumSubBits;
numSameBits += kNumSubBits;
}
node->NumSameBits = CSameBitsType(numSameBits);
node->Descendants[bitsNew].MatchPointer = pos + kMatchStartValue;
node->Descendants[bitsOld].MatchPointer = matchPointer;
return fullMatchLen;
}
const Byte *baseCurrentBytePointer = _buffer + pos;
const Byte *currentBytePointer = baseCurrentBytePointer;
UInt32 numLoadedBits = 0;
Byte curByte = 0;
CIndex *nodePointerPointer = &hashDescendant.NodePointer;
CNodePointer node = &m_Nodes[*nodePointerPointer];
distances += kNumHashBytes;
const Byte *bytePointerLimit = baseCurrentBytePointer + currentLimit;
const Byte *currentAddingOffset = _buffer;
#ifdef __AUTO_REMOVE
UInt32 lowPos;
if (pos > _sizeHistory)
lowPos = pos - _sizeHistory;
else
lowPos = 0;
#endif
while(true)
{
#ifdef __AUTO_REMOVE
if (node->LastMatch < lowPos)
{
RemoveNode(*nodePointerPointer);
*nodePointerPointer = pos + kMatchStartValue;
if (currentBytePointer == baseCurrentBytePointer)
return kPrevHashSize;
return kNumHashBytes + (UInt32)(currentBytePointer - baseCurrentBytePointer - 1);
}
#endif
if(numLoadedBits == 0)
{
*distances++ = pos - node->LastMatch - 1;
if(currentBytePointer >= bytePointerLimit)
{
for (UInt32 i = 0; i < kNumSubNodes; i++)
node->Descendants[i].MatchPointer = pos + kMatchStartValue;
node->LastMatch = pos;
node->NumSameBits = 0;
return fullCurrentLimit;
}
curByte = (*currentBytePointer++);
currentAddingOffset++;
numLoadedBits = MY_BYTE_SIZE;
}
UInt32 numSameBits = node->NumSameBits;
if(numSameBits > 0)
{
Byte byteXOR = ((*(currentAddingOffset + node->LastMatch -1)) >>
(MY_BYTE_SIZE - numLoadedBits)) ^ curByte;
while(numLoadedBits <= numSameBits)
{
if(byteXOR != 0)
{
AddInternalNode(node, nodePointerPointer, curByte, byteXOR,
numSameBits, pos);
return kNumHashBytes + (UInt32)(currentBytePointer - baseCurrentBytePointer - 1);
}
*distances++ = pos - node->LastMatch - 1;
numSameBits -= numLoadedBits;
if(currentBytePointer >= bytePointerLimit)
{
for (UInt32 i = 0; i < kNumSubNodes; i++)
node->Descendants[i].MatchPointer = pos + kMatchStartValue;
node->LastMatch = pos;
node->NumSameBits = CSameBitsType(node->NumSameBits - numSameBits);
return fullCurrentLimit;
}
numLoadedBits = MY_BYTE_SIZE;
curByte = (*currentBytePointer++);
byteXOR = curByte ^ (*(currentAddingOffset + node->LastMatch));
currentAddingOffset++;
}
if((byteXOR & ((1 << numSameBits) - 1)) != 0)
{
AddInternalNode(node, nodePointerPointer, curByte, byteXOR,
numSameBits, pos);
return kNumHashBytes + (UInt32)(currentBytePointer - baseCurrentBytePointer - 1);
}
curByte >>= numSameBits;
numLoadedBits -= numSameBits;
}
UInt32 descendantIndex = (curByte & kSubNodesMask);
numLoadedBits -= kNumSubBits;
nodePointerPointer = &node->Descendants[descendantIndex].NodePointer;
UInt32 nextNodeIndex = *nodePointerPointer;
node->LastMatch = pos;
if (nextNodeIndex < kDescendantEmptyValue)
{
curByte >>= kNumSubBits;
node = &m_Nodes[nextNodeIndex];
}
else if (nextNodeIndex == kDescendantEmptyValue)
{
node->Descendants[descendantIndex].MatchPointer = pos + kMatchStartValue;
return kNumHashBytes + (UInt32)(currentBytePointer - baseCurrentBytePointer - 1);
}
else
break;
}
UInt32 descendantIndex = (curByte & kSubNodesMask);
curByte >>= kNumSubBits;
CMatchPointer matchPointer = node->Descendants[descendantIndex].MatchPointer;
CMatchPointer realMatchPointer;
realMatchPointer = matchPointer - kMatchStartValue;
#ifdef __AUTO_REMOVE
if (realMatchPointer < lowPos)
{
node->Descendants[descendantIndex].MatchPointer = pos + kMatchStartValue;
return kNumHashBytes + (UInt32)(currentBytePointer - baseCurrentBytePointer - 1);
}
#endif
Byte byteXOR;
UInt32 numSameBits = 0;
if(numLoadedBits != 0)
{
Byte matchByte = *(currentAddingOffset + realMatchPointer -1);
matchByte >>= (MY_BYTE_SIZE - numLoadedBits);
byteXOR = matchByte ^ curByte;
if(byteXOR != 0)
{
AddLeafNode(node, curByte, byteXOR, numSameBits, pos, descendantIndex);
return kNumHashBytes + (UInt32)(currentBytePointer - baseCurrentBytePointer - 1);
}
numSameBits += numLoadedBits;
}
const Byte *matchBytePointer = _buffer + realMatchPointer +
(currentBytePointer - baseCurrentBytePointer);
for(; currentBytePointer < bytePointerLimit; numSameBits += MY_BYTE_SIZE)
{
curByte = (*currentBytePointer++);
*distances++ = pos - realMatchPointer - 1;
byteXOR = curByte ^ (*matchBytePointer++);
if(byteXOR != 0)
{
AddLeafNode(node, curByte, byteXOR, numSameBits, pos, descendantIndex);
return kNumHashBytes + (UInt32)(currentBytePointer - baseCurrentBytePointer - 1);
}
}
*distances = pos - realMatchPointer - 1;
node->Descendants[descendantIndex].MatchPointer = pos + kMatchStartValue;
if(*distances == 0)
{
m_SpecialMode = true;
m_NumNotChangedCycles = 0;
}
return fullCurrentLimit;
}
STDMETHODIMP_(void) CPatricia::DummyLongestMatch()
{
GetLongestMatch(m_TmpBacks);
}
// ------------------------------------
// Remove Match
typedef Byte CRemoveDataWord;
static const int kSizeRemoveDataWordInBits = MY_BYTE_SIZE * sizeof(CRemoveDataWord);
#ifndef __AUTO_REMOVE
void CPatricia::RemoveMatch()
{
if(m_SpecialRemoveMode)
{
if(GetIndexByte(_matchMaxLen - 1 - _sizeHistory) ==
GetIndexByte(_matchMaxLen - _sizeHistory))
return;
m_SpecialRemoveMode = false;
}
UInt32 pos = _pos + kNumHashBytes - _sizeHistory;
#ifdef __HASH_3
const Byte *pp = _buffer + _pos - _sizeHistory;
UInt32 hash2Value = ((UInt32(pp[0])) << 8) | pp[1];
UInt32 hashValue = (hash2Value << 8) | pp[2];
CDescendant &hashDescendant = m_HashDescendants[hashValue];
CDescendant &hash2Descendant = m_Hash2Descendants[hash2Value];
if (hash2Descendant >= kMatchStartValue2)
if(hash2Descendant.MatchPointer == pos + kMatchStartValue2)
hash2Descendant.MatchPointer = kDescendantEmptyValue2;
#else
UInt32 hashValue = UInt32(GetIndexByte(1 - _sizeHistory)) |
(UInt32(GetIndexByte(0 - _sizeHistory)) << 8);
CDescendant &hashDescendant = m_HashDescendants[hashValue];
#endif
if(hashDescendant.IsEmpty())
return;
if(hashDescendant.IsMatch())
{
if(hashDescendant.MatchPointer == pos + kMatchStartValue)
hashDescendant.MakeEmpty();
return;
}
UInt32 descendantIndex;
const CRemoveDataWord *currentPointer = (const CRemoveDataWord *)(_buffer + pos);
UInt32 numLoadedBits = 0;
CRemoveDataWord curWord = 0; // = 0 to disable GCC warning
CIndex *nodePointerPointer = &hashDescendant.NodePointer;
CNodePointer node = &m_Nodes[hashDescendant.NodePointer];
while(true)
{
if(numLoadedBits == 0)
{
curWord = *currentPointer++;
numLoadedBits = kSizeRemoveDataWordInBits;
}
UInt32 numSameBits = node->NumSameBits;
if(numSameBits > 0)
{
if (numLoadedBits <= numSameBits)
{
numSameBits -= numLoadedBits;
currentPointer += (numSameBits / kSizeRemoveDataWordInBits);
numSameBits %= kSizeRemoveDataWordInBits;
curWord = *currentPointer++;
numLoadedBits = kSizeRemoveDataWordInBits;
}
curWord >>= numSameBits;
numLoadedBits -= numSameBits;
}
descendantIndex = (curWord & kSubNodesMask);
numLoadedBits -= kNumSubBits;
curWord >>= kNumSubBits;
UInt32 nextNodeIndex = node->Descendants[descendantIndex].NodePointer;
if (nextNodeIndex < kDescendantEmptyValue)
{
nodePointerPointer = &node->Descendants[descendantIndex].NodePointer;
node = &m_Nodes[nextNodeIndex];
}
else
break;
}
if (node->Descendants[descendantIndex].MatchPointer != pos + kMatchStartValue)
{
const Byte *currentBytePointer = _buffer + _pos - _sizeHistory;
const Byte *currentBytePointerLimit = currentBytePointer + _matchMaxLen;
for(;currentBytePointer < currentBytePointerLimit; currentBytePointer++)
if(*currentBytePointer != *(currentBytePointer+1))
return;
m_SpecialRemoveMode = true;
return;
}
UInt32 numNodes = 0, numMatches = 0;
UInt32 i;
for (i = 0; i < kNumSubNodes; i++)
{
UInt32 nodeIndex = node->Descendants[i].NodePointer;
if (nodeIndex < kDescendantEmptyValue)
numNodes++;
else if (nodeIndex > kDescendantEmptyValue)
numMatches++;
}
numMatches -= 1;
if (numNodes + numMatches > 1)
{
node->Descendants[descendantIndex].MakeEmpty();
return;
}
if(numNodes == 1)
{
UInt32 i;
for (i = 0; i < kNumSubNodes; i++)
if (node->Descendants[i].IsNode())
break;
UInt32 nextNodeIndex = node->Descendants[i].NodePointer;
CNodePointer nextNode = &m_Nodes[nextNodeIndex];
nextNode->NumSameBits += node->NumSameBits + kNumSubBits;
*node = *nextNode;
nextNode->NextFreeNode = m_FreeNode;
m_FreeNode = nextNodeIndex;
return;
}
UInt32 matchPointer = 0; // = 0 to disable GCC warning
for (i = 0; i < kNumSubNodes; i++)
if (node->Descendants[i].IsMatch() && i != descendantIndex)
{
matchPointer = node->Descendants[i].MatchPointer;
break;
}
node->NextFreeNode = m_FreeNode;
m_FreeNode = *nodePointerPointer;
*nodePointerPointer = matchPointer;
}
#endif
// Commented code is more correct, but it gives warning
// on GCC: (1 << 32)
// So we use kMatchStartValue twice:
// kMatchStartValue = UInt32(1) << (kNumBitsInIndex - 1);
// must be defined in Pat.h
/*
const UInt32 kNormalizeStartPos = (UInt32(1) << (kNumBitsInIndex)) -
kMatchStartValue - kNumHashBytes - 1;
*/
const UInt32 kNormalizeStartPos = kMatchStartValue - kNumHashBytes - 1;
STDMETHODIMP CPatricia::MovePos()
{
#ifndef __AUTO_REMOVE
if(_pos >= _sizeHistory)
RemoveMatch();
#endif
RINOK(CLZInWindow::MovePos());
#ifdef __AUTO_REMOVE
if (m_NumUsedNodes >= m_NumNodes)
TestRemoveNodes();
#endif
if (_pos >= kNormalizeStartPos)
{
#ifdef __AUTO_REMOVE
TestRemoveNodesAndNormalize();
#else
Normalize();
#endif
}
return S_OK;
}
#ifndef __AUTO_REMOVE
void CPatricia::NormalizeDescendant(CDescendant &descendant, UInt32 subValue)
{
if (descendant.IsEmpty())
return;
if (descendant.IsMatch())
descendant.MatchPointer = descendant.MatchPointer - subValue;
else
{
CNode &node = m_Nodes[descendant.NodePointer];
node.LastMatch = node.LastMatch - subValue;
for (UInt32 i = 0; i < kNumSubNodes; i++)
NormalizeDescendant(node.Descendants[i], subValue);
}
}
void CPatricia::Normalize()
{
UInt32 subValue = _pos - _sizeHistory;
CLZInWindow::ReduceOffsets(subValue);
#ifdef __HASH_3
for(UInt32 hash = 0; hash < kHash2Size; hash++)
{
CDescendant &descendant = m_Hash2Descendants[hash];
if (descendant.MatchPointer != kDescendantsNotInitilized2)
{
UInt32 base = hash << 8;
for (UInt32 i = 0; i < 0x100; i++)
NormalizeDescendant(m_HashDescendants[base + i], subValue);
}
if (descendant.MatchPointer < kMatchStartValue2)
continue;
descendant.MatchPointer = descendant.MatchPointer - subValue;
}
#else
for(UInt32 hash = 0; hash < kHashSize; hash++)
NormalizeDescendant(m_HashDescendants[hash], subValue);
#endif
}
#else
void CPatricia::TestRemoveDescendant(CDescendant &descendant, UInt32 limitPos)
{
CNode &node = m_Nodes[descendant.NodePointer];
UInt32 numChilds = 0;
UInt32 childIndex = 0; // = 0 to disable GCC warning
for (UInt32 i = 0; i < kNumSubNodes; i++)
{
CDescendant &descendant2 = node.Descendants[i];
if (descendant2.IsEmpty())
continue;
if (descendant2.IsMatch())
{
if (descendant2.MatchPointer < limitPos)
descendant2.MakeEmpty();
else
{
numChilds++;
childIndex = i;
}
}
else
{
TestRemoveDescendant(descendant2, limitPos);
if (!descendant2.IsEmpty())
{
numChilds++;
childIndex = i;
}
}
}
if (numChilds > 1)
return;
CIndex nodePointerTemp = descendant.NodePointer;
if (numChilds == 1)
{
const CDescendant &descendant2 = node.Descendants[childIndex];
if (descendant2.IsNode())
m_Nodes[descendant2.NodePointer].NumSameBits += node.NumSameBits + kNumSubBits;
descendant = descendant2;
}
else
descendant.MakeEmpty();
node.NextFreeNode = m_FreeNode;
m_FreeNode = nodePointerTemp;
m_NumUsedNodes--;
}
void CPatricia::RemoveNode(UInt32 index)
{
CNode &node = m_Nodes[index];
for (UInt32 i = 0; i < kNumSubNodes; i++)
{
CDescendant &descendant2 = node.Descendants[i];
if (descendant2.IsNode())
RemoveNode(descendant2.NodePointer);
}
node.NextFreeNode = m_FreeNode;
m_FreeNode = index;
m_NumUsedNodes--;
}
void CPatricia::TestRemoveNodes()
{
UInt32 limitPos = kMatchStartValue + _pos - _sizeHistory + kNumHashBytes;
#ifdef __HASH_3
UInt32 limitPos2 = kMatchStartValue2 + _pos - _sizeHistory + kNumHashBytes;
for(UInt32 hash = 0; hash < kHash2Size; hash++)
{
CDescendant &descendant = m_Hash2Descendants[hash];
if (descendant.MatchPointer != kDescendantsNotInitilized2)
{
UInt32 base = hash << 8;
for (UInt32 i = 0; i < 0x100; i++)
{
CDescendant &descendant = m_HashDescendants[base + i];
if (descendant.IsEmpty())
continue;
if (descendant.IsMatch())
{
if (descendant.MatchPointer < limitPos)
descendant.MakeEmpty();
}
else
TestRemoveDescendant(descendant, limitPos);
}
}
if (descendant.MatchPointer < kMatchStartValue2)
continue;
if (descendant.MatchPointer < limitPos2)
descendant.MatchPointer = kDescendantEmptyValue2;
}
#else
for(UInt32 hash = 0; hash < kHashSize; hash++)
{
CDescendant &descendant = m_HashDescendants[hash];
if (descendant.IsEmpty())
continue;
if (descendant.IsMatch())
{
if (descendant.MatchPointer < limitPos)
descendant.MakeEmpty();
}
else
TestRemoveDescendant(descendant, limitPos);
}
#endif
}
void CPatricia::TestRemoveAndNormalizeDescendant(CDescendant &descendant,
UInt32 limitPos, UInt32 subValue)
{
if (descendant.IsEmpty())
return;
if (descendant.IsMatch())
{
if (descendant.MatchPointer < limitPos)
descendant.MakeEmpty();
else
descendant.MatchPointer = descendant.MatchPointer - subValue;
return;
}
CNode &node = m_Nodes[descendant.NodePointer];
UInt32 numChilds = 0;
UInt32 childIndex = 0; // = 0 to disable GCC warning
for (UInt32 i = 0; i < kNumSubNodes; i++)
{
CDescendant &descendant2 = node.Descendants[i];
TestRemoveAndNormalizeDescendant(descendant2, limitPos, subValue);
if (!descendant2.IsEmpty())
{
numChilds++;
childIndex = i;
}
}
if (numChilds > 1)
{
node.LastMatch = node.LastMatch - subValue;
return;
}
CIndex nodePointerTemp = descendant.NodePointer;
if (numChilds == 1)
{
const CDescendant &descendant2 = node.Descendants[childIndex];
if (descendant2.IsNode())
m_Nodes[descendant2.NodePointer].NumSameBits += node.NumSameBits + kNumSubBits;
descendant = descendant2;
}
else
descendant.MakeEmpty();
node.NextFreeNode = m_FreeNode;
m_FreeNode = nodePointerTemp;
m_NumUsedNodes--;
}
void CPatricia::TestRemoveNodesAndNormalize()
{
UInt32 subValue = _pos - _sizeHistory;
UInt32 limitPos = kMatchStartValue + _pos - _sizeHistory + kNumHashBytes;
CLZInWindow::ReduceOffsets(subValue);
#ifdef __HASH_3
UInt32 limitPos2 = kMatchStartValue2 + _pos - _sizeHistory + kNumHashBytes;
for(UInt32 hash = 0; hash < kHash2Size; hash++)
{
CDescendant &descendant = m_Hash2Descendants[hash];
if (descendant.MatchPointer != kDescendantsNotInitilized2)
{
UInt32 base = hash << 8;
for (UInt32 i = 0; i < 0x100; i++)
TestRemoveAndNormalizeDescendant(m_HashDescendants[base + i], limitPos, subValue);
}
if (descendant.MatchPointer < kMatchStartValue2)
continue;
if (descendant.MatchPointer < limitPos2)
descendant.MatchPointer = kDescendantEmptyValue2;
else
descendant.MatchPointer = descendant.MatchPointer - subValue;
}
#else
for(UInt32 hash = 0; hash < kHashSize; hash++)
TestRemoveAndNormalizeDescendant(m_HashDescendants[hash], limitPos, subValue);
#endif
}
#endif
STDMETHODIMP_(Byte) CPatricia::GetIndexByte(Int32 index)
{
return CLZInWindow::GetIndexByte(index);
}
STDMETHODIMP_(UInt32) CPatricia::GetMatchLen(Int32 index, UInt32 back, UInt32 limit)
{
return CLZInWindow::GetMatchLen(index, back, limit);
}
STDMETHODIMP_(UInt32) CPatricia::GetNumAvailableBytes()
{
return CLZInWindow::GetNumAvailableBytes();
}
STDMETHODIMP_(const Byte *) CPatricia::GetPointerToCurrentPos()
{
return CLZInWindow::GetPointerToCurrentPos();
}
}