xevion/easy7zip
1
0
Fork 0
mirror of https://github.com/Xevion/easy7zip.git synced 2025-12-09 02:07:06 -06:00
Code Issues Packages Projects Releases Wiki Activity

Update Brotli to v1.0.1

Browse Source 
- update README.md to the new versions (LZ4, Brotli)
This commit is contained in:
Tino Reichardt
2017-09-28 09:33:27 +02:00
parent 699a608314
commit a818384b05
39 changed files with 463 additions and 432 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
C/7zVersion.h
View File

@@ -1,7 +1,7 @@
#define MY_VER_MAJOR 17
#define MY_VER_MINOR 01
#define MY_VER_BUILD 0
#define MY_VERSION_NUMBERS "17.01 ZS v1.3.1 R1"
#define MY_VERSION_NUMBERS "17.01 ZS v1.3.1 R2"
#define MY_VERSION MY_VERSION_NUMBERS
#ifdef MY_CPU_NAME
@@ -10,7 +10,7 @@
#define MY_VERSION_CPU MY_VERSION
#endif
#define MY_DATE "2017-08-28"
#define MY_DATE "2017-09-28"
#undef MY_COPYRIGHT
#undef MY_VERSION_COPYRIGHT_DATE
#define MY_AUTHOR_NAME "Igor Pavlov, Tino Reichardt"

4
C/7zVersionTr.h
View File

@@ -1,9 +1,9 @@
#define MY_VER_MAJOR 1
#define MY_VER_MINOR 3
#define MY_VER_BUILD 1
#define MY_VERSION_NUMBERS "1.3.1 R1"
#define MY_VERSION_NUMBERS "1.3.1 R2"
#define MY_VERSION MY_VERSION_NUMBERS
#define MY_DATE "2017-08-28"
#define MY_DATE "2017-09-28"
#undef MY_COPYRIGHT
#undef MY_VERSION_COPYRIGHT_DATE
#define MY_AUTHOR_NAME "Tino Reichardt"

8
C/brotli/Brotli-Adjust.sh Normal file → Executable file
View File

@@ -34,3 +34,11 @@ cd ../enc
sed -i 's|include "./|include "./enc/|g' *.c
sed -i 's|include "../common|include "./common/|g' *.c
for f in *.c; do mv $f ../br_$f; done
exit
# and also disable these msc warnings:
# pragma warning(disable : 4100)
# pragma warning(disable : 4127)
# pragma warning(disable : 4389)
# pragma warning(disable : 4701)

54
C/brotli/README.md
View File

@@ -8,24 +8,32 @@ and 2nd order context modeling, with a compression ratio comparable to the best
currently available general-purpose compression methods. It is similar in speed
with deflate but offers more dense compression.
The specification of the Brotli Compressed Data Format is defined in [RFC 7932](https://www.ietf.org/rfc/rfc7932.txt).
The specification of the Brotli Compressed Data Format is defined in [RFC 7932](https://tools.ietf.org/html/rfc7932).
Brotli is open-sourced under the MIT License, see the LICENSE file.
Brotli mailing list:
https://groups.google.com/forum/#!forum/brotli
[![Build Status](https://travis-ci.org/google/brotli.svg?branch=master)](https://travis-ci.org/google/brotli)
[![TravisCI Build Status](https://travis-ci.org/google/brotli.svg?branch=master)](https://travis-ci.org/google/brotli)
[![AppVeyor Build Status](https://ci.appveyor.com/api/projects/status/github/google/brotli?branch=master&svg=true)](https://ci.appveyor.com/project/szabadka/brotli)
### Build instructions
#### Make
#### Autotools-style CMake
To build and run tests, simply do:
[configure-cmake](https://github.com/nemequ/configure-cmake) is an
autotools-style configure script for CMake-based projects (not supported on Windows).
$ ./configure && make
The basic commands to build, test and install brotli are:
If you want to install brotli, use one of the more advanced build systems below.
$ mkdir out && cd out
$ ../configure-cmake
$ make
$ make test
$ make install
By default, debug binaries are built. To generate "release" `Makefile` specify `--disable-debug` option to `configure-cmake`.
#### Bazel
@@ -33,19 +41,13 @@ See [Bazel](http://www.bazel.build/)
#### CMake
The basic commands to build, test and install brotli are:
The basic commands to build and install brotli are:
$ mkdir out && cd out && ../configure-cmake && make
$ make test
$ make install
$ mkdir out && cd out
$ cmake -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=./installed ..
$ cmake --build . --config Release --target install
You can use other [CMake](https://cmake.org/) configuration. For example, to
build static libraries and use a custom installation directory:
$ mkdir out-static && \
cd out-static && \
../configure-cmake --disable-shared-libs --prefix='/my/prefix/dir/'
$ make install
You can use other [CMake](https://cmake.org/) configuration.
#### Premake5
@@ -53,12 +55,16 @@ See [Premake5](https://premake.github.io/)
#### Python
To install the Python module from source, run the following:
To install the latest release of the Python module, run the following:
$ python setup.py install
$ pip install brotli
See the [Python readme](python/README.md) for more details on testing
and development.
To install the tip-of-the-tree version, run:
$ pip install --upgrade git+https://github.com/google/brotli
See the [Python readme](python/README.md) for more details on installing
from source, development, and testing.
### Benchmarks
* [Squash Compression Benchmark](https://quixdb.github.io/squash-benchmark/) / [Unstable Squash Compression Benchmark](https://quixdb.github.io/squash-benchmark/unstable/)
@@ -66,6 +72,12 @@ and development.
* [Lzturbo Benchmark](https://sites.google.com/site/powturbo/home/benchmark)
### Related projects
> **Disclaimer:** Brotli authors take no responsibility for the third party projects mentioned in this section.
Independent [decoder](https://github.com/madler/brotli) implementation by Mark Adler, based entirely on format specification.
JavaScript port of brotli [decoder](https://github.com/devongovett/brotli.js). Could be used directly via `npm install brotli`
Hand ported [decoder / encoder](https://github.com/dominikhlbg/BrotliHaxe) in haxe by Dominik Homberger. Output source code: JavaScript, PHP, Python, Java and C#
7Zip [plugin](https://github.com/mcmilk/7-Zip-Zstd)

8
C/brotli/br_backward_references.c
View File

@@ -48,6 +48,8 @@ static BROTLI_INLINE size_t ComputeDistanceCode(size_t distance,
#define EXPAND_CAT(a, b) CAT(a, b)
#define CAT(a, b) a ## b
#define FN(X) EXPAND_CAT(X, HASHER())
#define EXPORT_FN(X) EXPAND_CAT(X, EXPAND_CAT(PREFIX(), HASHER()))
#define PREFIX() N
#define HASHER() H2
/* NOLINTNEXTLINE(build/include) */
@@ -94,6 +96,8 @@ static BROTLI_INLINE size_t ComputeDistanceCode(size_t distance,
#include "./enc/backward_references_inc.h"
#undef HASHER
#undef PREFIX
#undef EXPORT_FN
#undef FN
#undef CAT
#undef EXPAND_CAT
@@ -113,11 +117,11 @@ void BrotliCreateBackwardReferences(const BrotliDictionary* dictionary,
switch (params->hasher.type) {
#define CASE_(N) \
case N: \
CreateBackwardReferencesH ## N(dictionary, \
CreateBackwardReferencesNH ## N(dictionary, \
kStaticDictionaryHash, num_bytes, position, ringbuffer, \
ringbuffer_mask, params, hasher, dist_cache, \
last_insert_len, commands, num_commands, num_literals); \
break;
return;
FOR_GENERIC_HASHERS(CASE_)
#undef CASE_
default:

86
C/brotli/br_backward_references_hq.c
View File

@@ -300,6 +300,7 @@ static size_t ComputeMinimumCopyLength(const float start_cost,
static uint32_t ComputeDistanceShortcut(const size_t block_start,
const size_t pos,
const size_t max_backward,
const size_t gap,
const ZopfliNode* nodes) {
const size_t clen = ZopfliNodeCopyLength(&nodes[pos]);
const size_t ilen = nodes[pos].insert_length;
@@ -311,8 +312,8 @@ static uint32_t ComputeDistanceShortcut(const size_t block_start,
does not update the last distances. */
if (pos == 0) {
return 0;
} else if (dist + clen <= block_start + pos &&
dist <= max_backward &&
} else if (dist + clen <= block_start + pos + gap &&
dist <= max_backward + gap &&
ZopfliNodeDistanceCode(&nodes[pos]) > 0) {
return (uint32_t)pos;
} else {
@@ -350,12 +351,12 @@ static void ComputeDistanceCache(const size_t pos,
is eligible. */
static void EvaluateNode(
const size_t block_start, const size_t pos, const size_t max_backward_limit,
const int* starting_dist_cache, const ZopfliCostModel* model,
StartPosQueue* queue, ZopfliNode* nodes) {
const size_t gap, const int* starting_dist_cache,
const ZopfliCostModel* model, StartPosQueue* queue, ZopfliNode* nodes) {
/* Save cost, because ComputeDistanceCache invalidates it. */
float node_cost = nodes[pos].u.cost;
nodes[pos].u.shortcut = ComputeDistanceShortcut(
block_start, pos, max_backward_limit, nodes);
block_start, pos, max_backward_limit, gap, nodes);
if (node_cost <= ZopfliCostModelGetLiteralCosts(model, 0, pos)) {
PosData posdata;
posdata.pos = pos;
@@ -385,9 +386,10 @@ static size_t UpdateNodes(
size_t min_len;
size_t result = 0;
size_t k;
size_t gap = 0;
EvaluateNode(block_start, pos, max_backward_limit, starting_dist_cache, model,
queue, nodes);
EvaluateNode(block_start, pos, max_backward_limit, gap, starting_dist_cache,
model, queue, nodes);
{
const PosData* posdata = StartPosQueueAt(queue, 0);
@@ -415,25 +417,31 @@ static size_t UpdateNodes(
const size_t backward =
(size_t)(posdata->distance_cache[idx] + kDistanceCacheOffset[j]);
size_t prev_ix = cur_ix - backward;
if (prev_ix >= cur_ix) {
size_t len = 0;
uint8_t continuation = ringbuffer[cur_ix_masked + best_len];
if (cur_ix_masked + best_len > ringbuffer_mask) {
break;
}
if (BROTLI_PREDICT_FALSE(backward > max_distance + gap)) {
continue;
}
if (BROTLI_PREDICT_FALSE(backward > max_distance)) {
continue;
}
prev_ix &= ringbuffer_mask;
if (backward <= max_distance) {
if (prev_ix >= cur_ix) {
continue;
}
if (cur_ix_masked + best_len > ringbuffer_mask ||
prev_ix + best_len > ringbuffer_mask ||
ringbuffer[cur_ix_masked + best_len] !=
ringbuffer[prev_ix + best_len]) {
prev_ix &= ringbuffer_mask;
if (prev_ix + best_len > ringbuffer_mask ||
continuation != ringbuffer[prev_ix + best_len]) {
continue;
}
len = FindMatchLengthWithLimit(&ringbuffer[prev_ix],
&ringbuffer[cur_ix_masked],
max_len);
} else {
continue;
}
{
const size_t len =
FindMatchLengthWithLimit(&ringbuffer[prev_ix],
&ringbuffer[cur_ix_masked],
max_len);
const float dist_cost = base_cost +
ZopfliCostModelGetDistanceCost(model, j);
size_t l;
@@ -464,7 +472,8 @@ static size_t UpdateNodes(
for (j = 0; j < num_matches; ++j) {
BackwardMatch match = matches[j];
size_t dist = match.distance;
BROTLI_BOOL is_dictionary_match = TO_BROTLI_BOOL(dist > max_distance);
BROTLI_BOOL is_dictionary_match =
TO_BROTLI_BOOL(dist > max_distance + gap);
/* We already tried all possible last distance matches, so we can use
normal distance code here. */
size_t dist_code = dist + BROTLI_NUM_DISTANCE_SHORT_CODES - 1;
@@ -526,11 +535,14 @@ void BrotliZopfliCreateCommands(const size_t num_bytes,
const ZopfliNode* nodes,
int* dist_cache,
size_t* last_insert_len,
const BrotliEncoderParams* params,
Command* commands,
size_t* num_literals) {
size_t pos = 0;
uint32_t offset = nodes[0].u.next;
size_t i;
size_t gap = 0;
BROTLI_UNUSED(params);
for (i = 0; offset != BROTLI_UINT32_MAX; i++) {
const ZopfliNode* next = &nodes[pos + offset];
size_t copy_length = ZopfliNodeCopyLength(next);
@@ -546,11 +558,11 @@ void BrotliZopfliCreateCommands(const size_t num_bytes,
size_t len_code = ZopfliNodeLengthCode(next);
size_t max_distance =
BROTLI_MIN(size_t, block_start + pos, max_backward_limit);
BROTLI_BOOL is_dictionary = TO_BROTLI_BOOL(distance > max_distance);
BROTLI_BOOL is_dictionary = TO_BROTLI_BOOL(distance > max_distance + gap);
size_t dist_code = ZopfliNodeDistanceCode(next);
InitCommand(
&commands[i], insert_length, copy_length, len_code, dist_code);
InitCommand(&commands[i], insert_length,
copy_length, (int)len_code - (int)copy_length, dist_code);
if (!is_dictionary && dist_code > 0) {
dist_cache[3] = dist_cache[2];
@@ -572,6 +584,7 @@ static size_t ZopfliIterate(size_t num_bytes,
size_t ringbuffer_mask,
const BrotliEncoderParams* params,
const size_t max_backward_limit,
const size_t gap,
const int* dist_cache,
const ZopfliCostModel* model,
const uint32_t* num_matches,
@@ -600,8 +613,8 @@ static size_t ZopfliIterate(size_t num_bytes,
while (skip) {
i++;
if (i + 3 >= num_bytes) break;
EvaluateNode(
position, i, max_backward_limit, dist_cache, model, &queue, nodes);
EvaluateNode(position, i, max_backward_limit, gap, dist_cache, model,
&queue, nodes);
cur_match_pos += num_matches[i];
skip--;
}
@@ -629,6 +642,7 @@ size_t BrotliZopfliComputeShortestPath(MemoryManager* m,
const size_t store_end = num_bytes >= StoreLookaheadH10() ?
position + num_bytes - StoreLookaheadH10() + 1 : position;
size_t i;
size_t gap = 0;
nodes[0].length = 0;
nodes[0].u.cost = 0;
InitZopfliCostModel(m, &model, num_bytes);
@@ -640,7 +654,8 @@ size_t BrotliZopfliComputeShortestPath(MemoryManager* m,
const size_t pos = position + i;
const size_t max_distance = BROTLI_MIN(size_t, pos, max_backward_limit);
size_t num_matches = FindAllMatchesH10(hasher, dictionary, ringbuffer,
ringbuffer_mask, pos, num_bytes - i, max_distance, params, matches);
ringbuffer_mask, pos, num_bytes - i, max_distance, gap, params,
matches);
size_t skip;
if (num_matches > 0 &&
BackwardMatchLength(&matches[num_matches - 1]) > max_zopfli_len) {
@@ -662,8 +677,8 @@ size_t BrotliZopfliComputeShortestPath(MemoryManager* m,
while (skip) {
i++;
if (i + HashTypeLengthH10() - 1 >= num_bytes) break;
EvaluateNode(
position, i, max_backward_limit, dist_cache, &model, &queue, nodes);
EvaluateNode(position, i, max_backward_limit, gap, dist_cache, &model,
&queue, nodes);
skip--;
}
}
@@ -688,7 +703,7 @@ void BrotliCreateZopfliBackwardReferences(
dist_cache, hasher, nodes);
if (BROTLI_IS_OOM(m)) return;
BrotliZopfliCreateCommands(num_bytes, position, max_backward_limit, nodes,
dist_cache, last_insert_len, commands, num_literals);
dist_cache, last_insert_len, params, commands, num_literals);
BROTLI_FREE(m, nodes);
}
@@ -712,6 +727,7 @@ void BrotliCreateHqZopfliBackwardReferences(
ZopfliCostModel model;
ZopfliNode* nodes;
BackwardMatch* matches = BROTLI_ALLOC(m, BackwardMatch, matches_size);
size_t gap = 0;
if (BROTLI_IS_OOM(m)) return;
for (i = 0; i + HashTypeLengthH10() - 1 < num_bytes; ++i) {
const size_t pos = position + i;
@@ -725,14 +741,12 @@ void BrotliCreateHqZopfliBackwardReferences(
cur_match_pos + MAX_NUM_MATCHES_H10);
if (BROTLI_IS_OOM(m)) return;
num_found_matches = FindAllMatchesH10(hasher, dictionary, ringbuffer,
ringbuffer_mask, pos, max_length, max_distance, params,
ringbuffer_mask, pos, max_length, max_distance, gap, params,
&matches[cur_match_pos]);
cur_match_end = cur_match_pos + num_found_matches;
for (j = cur_match_pos; j + 1 < cur_match_end; ++j) {
assert(BackwardMatchLength(&matches[j]) <
assert(BackwardMatchLength(&matches[j]) <=
BackwardMatchLength(&matches[j + 1]));
assert(matches[j].distance > max_distance ||
matches[j].distance <= matches[j + 1].distance);
}
num_matches[i] = (uint32_t)num_found_matches;
if (num_found_matches > 0) {
@@ -774,10 +788,10 @@ void BrotliCreateHqZopfliBackwardReferences(
*last_insert_len = orig_last_insert_len;
memcpy(dist_cache, orig_dist_cache, 4 * sizeof(dist_cache[0]));
*num_commands += ZopfliIterate(num_bytes, position, ringbuffer,
ringbuffer_mask, params, max_backward_limit, dist_cache,
ringbuffer_mask, params, max_backward_limit, gap, dist_cache,
&model, num_matches, matches, nodes);
BrotliZopfliCreateCommands(num_bytes, position, max_backward_limit,
nodes, dist_cache, last_insert_len, commands, num_literals);
nodes, dist_cache, last_insert_len, params, commands, num_literals);
}
CleanupZopfliCostModel(m, &model);
BROTLI_FREE(m, nodes);

6
C/brotli/br_block_splitter.c
View File

@@ -74,11 +74,9 @@ static void CopyLiteralsToByteArray(const Command* cmds,
}
}
static BROTLI_INLINE unsigned int MyRand(unsigned int* seed) {
static BROTLI_INLINE uint32_t MyRand(uint32_t* seed) {
/* Initial seed should be 7. In this case, loop length is (1 << 29). */
*seed *= 16807U;
if (*seed == 0) {
*seed = 1;
}
return *seed;
}

11
C/brotli/br_brotli_bit_stream.c
View File

@@ -1325,17 +1325,6 @@ void BrotliStoreUncompressedMetaBlock(BROTLI_BOOL is_final_block,
}
}
void BrotliStoreSyncMetaBlock(size_t* BROTLI_RESTRICT storage_ix,
uint8_t* BROTLI_RESTRICT storage) {
/* Empty metadata meta-block bit pattern:
1 bit: is_last (0)
2 bits: num nibbles (3)
1 bit: reserved (0)
2 bits: metadata length bytes (0) */
BrotliWriteBits(6, 6, storage_ix, storage);
JumpToByteBoundary(storage_ix, storage);
}
#if defined(__cplusplus) || defined(c_plusplus)
} /* extern "C" */
#endif

6
C/brotli/br_compress_fragment.c
View File

@@ -42,7 +42,7 @@ extern "C" {
static const uint32_t kHashMul32 = 0x1e35a7bd;
static BROTLI_INLINE uint32_t Hash(const uint8_t* p, size_t shift) {
const uint64_t h = (BROTLI_UNALIGNED_LOAD64(p) << 24) * kHashMul32;
const uint64_t h = (BROTLI_UNALIGNED_LOAD64LE(p) << 24) * kHashMul32;
return (uint32_t)(h >> shift);
}
@@ -603,7 +603,7 @@ trawl:
compression we first update "table" with the hashes of some positions
within the last copy. */
{
uint64_t input_bytes = BROTLI_UNALIGNED_LOAD64(ip - 3);
uint64_t input_bytes = BROTLI_UNALIGNED_LOAD64LE(ip - 3);
uint32_t prev_hash = HashBytesAtOffset(input_bytes, 0, shift);
uint32_t cur_hash = HashBytesAtOffset(input_bytes, 3, shift);
table[prev_hash] = (int)(ip - base_ip - 3);
@@ -640,7 +640,7 @@ trawl:
compression we first update "table" with the hashes of some positions
within the last copy. */
{
uint64_t input_bytes = BROTLI_UNALIGNED_LOAD64(ip - 3);
uint64_t input_bytes = BROTLI_UNALIGNED_LOAD64LE(ip - 3);
uint32_t prev_hash = HashBytesAtOffset(input_bytes, 0, shift);
uint32_t cur_hash = HashBytesAtOffset(input_bytes, 3, shift);
table[prev_hash] = (int)(ip - base_ip - 3);

10
C/brotli/br_compress_fragment_two_pass.c
View File

@@ -41,7 +41,7 @@ extern "C" {
static const uint32_t kHashMul32 = 0x1e35a7bd;
static BROTLI_INLINE uint32_t Hash(const uint8_t* p, size_t shift) {
const uint64_t h = (BROTLI_UNALIGNED_LOAD64(p) << 16) * kHashMul32;
const uint64_t h = (BROTLI_UNALIGNED_LOAD64LE(p) << 16) * kHashMul32;
return (uint32_t)(h >> shift);
}
@@ -346,7 +346,7 @@ trawl:
/* We could immediately start working at ip now, but to improve
compression we first update "table" with the hashes of some
positions within the last copy. */
uint64_t input_bytes = BROTLI_UNALIGNED_LOAD64(ip - 5);
uint64_t input_bytes = BROTLI_UNALIGNED_LOAD64LE(ip - 5);
uint32_t prev_hash = HashBytesAtOffset(input_bytes, 0, shift);
uint32_t cur_hash;
table[prev_hash] = (int)(ip - base_ip - 5);
@@ -354,7 +354,7 @@ trawl:
table[prev_hash] = (int)(ip - base_ip - 4);
prev_hash = HashBytesAtOffset(input_bytes, 2, shift);
table[prev_hash] = (int)(ip - base_ip - 3);
input_bytes = BROTLI_UNALIGNED_LOAD64(ip - 2);
input_bytes = BROTLI_UNALIGNED_LOAD64LE(ip - 2);
cur_hash = HashBytesAtOffset(input_bytes, 2, shift);
prev_hash = HashBytesAtOffset(input_bytes, 0, shift);
table[prev_hash] = (int)(ip - base_ip - 2);
@@ -386,7 +386,7 @@ trawl:
/* We could immediately start working at ip now, but to improve
compression we first update "table" with the hashes of some
positions within the last copy. */
uint64_t input_bytes = BROTLI_UNALIGNED_LOAD64(ip - 5);
uint64_t input_bytes = BROTLI_UNALIGNED_LOAD64LE(ip - 5);
uint32_t prev_hash = HashBytesAtOffset(input_bytes, 0, shift);
uint32_t cur_hash;
table[prev_hash] = (int)(ip - base_ip - 5);
@@ -394,7 +394,7 @@ trawl:
table[prev_hash] = (int)(ip - base_ip - 4);
prev_hash = HashBytesAtOffset(input_bytes, 2, shift);
table[prev_hash] = (int)(ip - base_ip - 3);
input_bytes = BROTLI_UNALIGNED_LOAD64(ip - 2);
input_bytes = BROTLI_UNALIGNED_LOAD64LE(ip - 2);
cur_hash = HashBytesAtOffset(input_bytes, 2, shift);
prev_hash = HashBytesAtOffset(input_bytes, 0, shift);
table[prev_hash] = (int)(ip - base_ip - 2);

97
C/brotli/br_decode.c
View File

@@ -39,6 +39,11 @@ extern "C" {
#define HUFFMAN_TABLE_BITS 8U
#define HUFFMAN_TABLE_MASK 0xff
/* We need the slack region for the following reasons:
- doing up to two 16-byte copies for fast backward copying
- inserting transformed dictionary word (5 prefix + 24 base + 8 suffix) */
static const uint32_t kRingBufferWriteAheadSlack = 42;
static const uint8_t kCodeLengthCodeOrder[BROTLI_CODE_LENGTH_CODES] = {
1, 2, 3, 4, 0, 5, 17, 6, 16, 7, 8, 9, 10, 11, 12, 13, 14, 15,
};
@@ -52,6 +57,17 @@ static const uint8_t kCodeLengthPrefixValue[16] = {
0, 4, 3, 2, 0, 4, 3, 1, 0, 4, 3, 2, 0, 4, 3, 5,
};
BROTLI_BOOL BrotliDecoderSetParameter(
BrotliDecoderState* state, BrotliDecoderParameter p, uint32_t value) {
switch (p) {
case BROTLI_DECODER_PARAM_DISABLE_RING_BUFFER_REALLOCATION:
state->canny_ringbuffer_allocation = !!value ? 0 : 1;
return BROTLI_TRUE;
default: return BROTLI_FALSE;
}
}
BrotliDecoderState* BrotliDecoderCreateInstance(
brotli_alloc_func alloc_func, brotli_free_func free_func, void* opaque) {
BrotliDecoderState* state = 0;
@@ -66,7 +82,6 @@ BrotliDecoderState* BrotliDecoderCreateInstance(
}
BrotliDecoderStateInitWithCustomAllocators(
state, alloc_func, free_func, opaque);
state->error_code = BROTLI_DECODER_NO_ERROR;
return state;
}
@@ -1211,7 +1226,9 @@ static BrotliDecoderErrorCode BROTLI_NOINLINE WriteRingBuffer(
BROTLI_LOG_UINT(to_write);
BROTLI_LOG_UINT(num_written);
s->partial_pos_out += num_written;
if (total_out) *total_out = s->partial_pos_out - (size_t)s->custom_dict_size;
if (total_out) {
*total_out = s->partial_pos_out;
}
if (num_written < to_write) {
if (s->ringbuffer_size == (1 << s->window_bits) || force) {
return BROTLI_DECODER_NEEDS_MORE_OUTPUT;
@@ -1243,22 +1260,16 @@ static void BROTLI_NOINLINE WrapRingBuffer(BrotliDecoderState* s) {
Last two bytes of ring-buffer are initialized to 0, so context calculation
could be done uniformly for the first two and all other positions.
Custom dictionary, if any, is copied to the end of ring-buffer.
*/
static BROTLI_BOOL BROTLI_NOINLINE BrotliEnsureRingBuffer(
BrotliDecoderState* s) {
/* We need the slack region for the following reasons:
- doing up to two 16-byte copies for fast backward copying
- inserting transformed dictionary word (5 prefix + 24 base + 8 suffix) */
static const int kRingBufferWriteAheadSlack = 42;
uint8_t* old_ringbuffer = s->ringbuffer;
if (s->ringbuffer_size == s->new_ringbuffer_size) {
return BROTLI_TRUE;
}
s->ringbuffer = (uint8_t*)BROTLI_ALLOC(s, (size_t)(s->new_ringbuffer_size +
kRingBufferWriteAheadSlack));
s->ringbuffer = (uint8_t*)BROTLI_ALLOC(s, (size_t)(s->new_ringbuffer_size) +
kRingBufferWriteAheadSlack);
if (s->ringbuffer == 0) {
/* Restore previous value. */
s->ringbuffer = old_ringbuffer;
@@ -1267,13 +1278,7 @@ static BROTLI_BOOL BROTLI_NOINLINE BrotliEnsureRingBuffer(
s->ringbuffer[s->new_ringbuffer_size - 2] = 0;
s->ringbuffer[s->new_ringbuffer_size - 1] = 0;
if (!old_ringbuffer) {
if (s->custom_dict) {
memcpy(s->ringbuffer, s->custom_dict, (size_t)s->custom_dict_size);
s->partial_pos_out = (size_t)s->custom_dict_size;
s->pos = s->custom_dict_size;
}
} else {
if (!!old_ringbuffer) {
memcpy(s->ringbuffer, old_ringbuffer, (size_t)s->pos);
BROTLI_FREE(s, old_ringbuffer);
}
@@ -1362,16 +1367,20 @@ static void BROTLI_NOINLINE BrotliCalculateRingBufferSize(
}
if (!s->ringbuffer) {
/* Custom dictionary counts as a "virtual" output. */
output_size = s->custom_dict_size;
output_size = 0;
} else {
output_size = s->pos;
}
output_size += s->meta_block_remaining_len;
min_size = min_size < output_size ? output_size : min_size;
while ((new_ringbuffer_size >> 1) >= min_size) {
new_ringbuffer_size >>= 1;
if (!!s->canny_ringbuffer_allocation) {
/* Reduce ring buffer size to save memory when server is unscrupulous.
In worst case memory usage might be 1.5x bigger for a short period of
ring buffer reallocation.*/
while ((new_ringbuffer_size >> 1) >= min_size) {
new_ringbuffer_size >>= 1;
}
}
s->new_ringbuffer_size = new_ringbuffer_size;
@@ -1736,17 +1745,20 @@ CommandPostDecodeLiterals:
/* Apply copy of LZ77 back-reference, or static dictionary reference if
the distance is larger than the max LZ77 distance */
if (s->distance_code > s->max_distance) {
int address = s->distance_code - s->max_distance - 1;
if (i >= BROTLI_MIN_DICTIONARY_WORD_LENGTH &&
i <= BROTLI_MAX_DICTIONARY_WORD_LENGTH) {
int offset = (int)s->dictionary->offsets_by_length[i];
int word_id = s->distance_code - s->max_distance - 1;
uint32_t shift = s->dictionary->size_bits_by_length[i];
int mask = (int)BitMask(shift);
int word_idx = word_id & mask;
int transform_idx = word_id >> shift;
int word_idx = address & mask;
int transform_idx = address >> shift;
/* Compensate double distance-ring-buffer roll. */
s->dist_rb_idx += s->distance_context;
offset += word_idx * i;
if (BROTLI_PREDICT_FALSE(!s->dictionary->data)) {
return BROTLI_FAILURE(BROTLI_DECODER_ERROR_DICTIONARY_NOT_SET);
}
if (transform_idx < kNumTransforms) {
const uint8_t* word = &s->dictionary->data[offset];
int len = i;
@@ -1899,6 +1911,10 @@ BrotliDecoderResult BrotliDecoderDecompressStream(
size_t* available_out, uint8_t** next_out, size_t* total_out) {
BrotliDecoderErrorCode result = BROTLI_DECODER_SUCCESS;
BrotliBitReader* br = &s->br;
/* Do not try to process further in a case of unrecoverable error. */
if ((int)s->error_code < 0) {
return BROTLI_DECODER_RESULT_ERROR;
}
if (*available_out && (!next_out || !*next_out)) {
return SaveErrorCode(
s, BROTLI_FAILURE(BROTLI_DECODER_ERROR_INVALID_ARGUMENTS));
@@ -1919,7 +1935,13 @@ BrotliDecoderResult BrotliDecoderDecompressStream(
if (result != BROTLI_DECODER_SUCCESS) { /* Error, needs more input/output */
if (result == BROTLI_DECODER_NEEDS_MORE_INPUT) {
if (s->ringbuffer != 0) { /* Pro-actively push output. */
WriteRingBuffer(s, available_out, next_out, total_out, BROTLI_TRUE);
BrotliDecoderErrorCode intermediate_result = WriteRingBuffer(s,
available_out, next_out, total_out, BROTLI_TRUE);
/* WriteRingBuffer checks s->meta_block_remaining_len validity. */
if ((int)intermediate_result < 0) {
result = intermediate_result;
break;
}
}
if (s->buffer_length != 0) { /* Used with internal buffer. */
if (br->avail_in == 0) { /* Successfully finished read transaction. */
@@ -1993,11 +2015,6 @@ BrotliDecoderResult BrotliDecoderDecompressStream(
}
/* Maximum distance, see section 9.1. of the spec. */
s->max_backward_distance = (1 << s->window_bits) - BROTLI_WINDOW_GAP;
/* Limit custom dictionary size. */
if (s->custom_dict_size >= s->max_backward_distance) {
s->custom_dict += s->custom_dict_size - s->max_backward_distance;
s->custom_dict_size = s->max_backward_distance;
}
/* Allocate memory for both block_type_trees and block_len_trees. */
s->block_type_trees = (HuffmanCode*)BROTLI_ALLOC(s,
@@ -2294,16 +2311,11 @@ BrotliDecoderResult BrotliDecoderDecompressStream(
return SaveErrorCode(s, result);
}
void BrotliDecoderSetCustomDictionary(
BrotliDecoderState* s, size_t size, const uint8_t* dict) {
if (size > (1u << 24)) {
return;
}
s->custom_dict = dict;
s->custom_dict_size = (int)size;
}
BROTLI_BOOL BrotliDecoderHasMoreOutput(const BrotliDecoderState* s) {
/* After unrecoverable error remaining output is considered nonsensical. */
if ((int)s->error_code < 0) {
return BROTLI_FALSE;
}
return TO_BROTLI_BOOL(
s->ringbuffer != 0 && UnwrittenBytes(s, BROTLI_FALSE) != 0);
}
@@ -2313,17 +2325,20 @@ const uint8_t* BrotliDecoderTakeOutput(BrotliDecoderState* s, size_t* size) {
size_t available_out = *size ? *size : 1u << 24;
size_t requested_out = available_out;
BrotliDecoderErrorCode status;
if (s->ringbuffer == 0) {
if ((s->ringbuffer == 0) || ((int)s->error_code < 0)) {
*size = 0;
return 0;
}
WrapRingBuffer(s);
status = WriteRingBuffer(s, &available_out, &result, 0, BROTLI_TRUE);
/* Either WriteRingBuffer returns those "success" codes... */
if (status == BROTLI_DECODER_SUCCESS ||
status == BROTLI_DECODER_NEEDS_MORE_OUTPUT) {
*size = requested_out - available_out;
} else {
/* This might happen if previous decoder error code was ignored. */
/* ... or stream is broken. Normally this should be caught by
BrotliDecoderDecompressStream, this is just a safeguard. */
if ((int)status < 0) SaveErrorCode(s, status);
*size = 0;
result = 0;
}

55
C/brotli/br_dictionary.c
View File

@@ -10,24 +10,8 @@
extern "C" {
#endif
static const BrotliDictionary kBrotliDictionary = {
/* size_bits_by_length */
{
0, 0, 0, 0, 10, 10, 11, 11,
10, 10, 10, 10, 10, 9, 9, 8,
7, 7, 8, 7, 7, 6, 6, 5,
5, 0, 0, 0, 0, 0, 0, 0
},
/* offsets_by_length */
{
0, 0, 0, 0, 0, 4096, 9216, 21504,
35840, 44032, 53248, 63488, 74752, 87040, 93696, 100864,
104704, 106752, 108928, 113536, 115968, 118528, 119872, 121280,
122016, 122784, 122784, 122784, 122784, 122784, 122784, 122784
},
/* data */
#ifndef BROTLI_EXTERNAL_DICTIONARY_DATA
static const uint8_t kBrotliDictionaryData[] =
{
116,105,109,101,100,111,119,110,108,105,102,101,108,101,102,116,98,97,99,107,99,
111,100,101,100,97,116,97,115,104,111,119,111,110,108,121,115,105,116,101,99,105
@@ -5875,12 +5859,47 @@ static const BrotliDictionary kBrotliDictionary = {
,164,181,224,164,190,224,164,136,224,164,184,224,164,149,224,165,141,224,164,176
,224,164,191,224,164,175,224,164,164,224,164,190
}
;
#endif /* !BROTLI_EXTERNAL_DICTIONARY_DATA */
static BrotliDictionary kBrotliDictionary = {
/* size_bits_by_length */
{
0, 0, 0, 0, 10, 10, 11, 11,
10, 10, 10, 10, 10, 9, 9, 8,
7, 7, 8, 7, 7, 6, 6, 5,
5, 0, 0, 0, 0, 0, 0, 0
},
/* offsets_by_length */
{
0, 0, 0, 0, 0, 4096, 9216, 21504,
35840, 44032, 53248, 63488, 74752, 87040, 93696, 100864,
104704, 106752, 108928, 113536, 115968, 118528, 119872, 121280,
122016, 122784, 122784, 122784, 122784, 122784, 122784, 122784
},
/* data_size == sizeof(kBrotliDictionaryData) */
122784,
/* data */
#ifdef BROTLI_EXTERNAL_DICTIONARY_DATA
NULL
#else
kBrotliDictionaryData
#endif
};
const BrotliDictionary* BrotliGetDictionary() {
return &kBrotliDictionary;
}
void BrotliSetDictionaryData(const uint8_t* data) {
if (!!data && !kBrotliDictionary.data) {
kBrotliDictionary.data = data;
}
}
#if defined(__cplusplus) || defined(c_plusplus)
} /* extern "C" */
#endif

45
C/brotli/br_encode.c
View File

@@ -385,8 +385,7 @@ static void ChooseContextMap(int quality,
context values, based on the entropy reduction of histograms over the
first 5 bits of literals. */
static BROTLI_BOOL ShouldUseComplexStaticContextMap(const uint8_t* input,
size_t start_pos, size_t length, size_t mask, int quality,
size_t size_hint, ContextType* literal_context_mode,
size_t start_pos, size_t length, size_t mask, int quality, size_t size_hint,
size_t* num_literal_contexts, const uint32_t** literal_context_map) {
static const uint32_t kStaticContextMapComplexUTF8[64] = {
11, 11, 12, 12, /* 0 special */
@@ -457,7 +456,6 @@ static BROTLI_BOOL ShouldUseComplexStaticContextMap(const uint8_t* input,
if (entropy[2] > 3.0 || entropy[1] - entropy[2] < 0.2) {
return BROTLI_FALSE;
} else {
*literal_context_mode = CONTEXT_UTF8;
*num_literal_contexts = 13;
*literal_context_map = kStaticContextMapComplexUTF8;
return BROTLI_TRUE;
@@ -466,13 +464,12 @@ static BROTLI_BOOL ShouldUseComplexStaticContextMap(const uint8_t* input,
}
static void DecideOverLiteralContextModeling(const uint8_t* input,
size_t start_pos, size_t length, size_t mask, int quality,
size_t size_hint, ContextType* literal_context_mode,
size_t start_pos, size_t length, size_t mask, int quality, size_t size_hint,
size_t* num_literal_contexts, const uint32_t** literal_context_map) {
if (quality < MIN_QUALITY_FOR_CONTEXT_MODELING || length < 64) {
return;
} else if (ShouldUseComplexStaticContextMap(
input, start_pos, length, mask, quality, size_hint, literal_context_mode,
input, start_pos, length, mask, quality, size_hint,
num_literal_contexts, literal_context_map)) {
/* Context map was already set, nothing else to do. */
} else {
@@ -492,7 +489,6 @@ static void DecideOverLiteralContextModeling(const uint8_t* input,
prev = lut[literal >> 6] * 3;
}
}
*literal_context_mode = CONTEXT_UTF8;
ChooseContextMap(quality, &bigram_prefix_histo[0], num_literal_contexts,
literal_context_map);
}
@@ -596,7 +592,7 @@ static void WriteMetaBlockInternal(MemoryManager* m,
if (!params->disable_literal_context_modeling) {
DecideOverLiteralContextModeling(
data, wrapped_last_flush_pos, bytes, mask, params->quality,
params->size_hint, &literal_context_mode, &num_literal_contexts,
params->size_hint, &num_literal_contexts,
&literal_context_map);
}
BrotliBuildMetaBlockGreedy(m, data, wrapped_last_flush_pos, mask,
@@ -832,36 +828,6 @@ static void CopyInputToRingBuffer(BrotliEncoderState* s,
}
}
void BrotliEncoderSetCustomDictionary(BrotliEncoderState* s, size_t size,
const uint8_t* dict) {
size_t max_dict_size = BROTLI_MAX_BACKWARD_LIMIT(s->params.lgwin);
size_t dict_size = size;
MemoryManager* m = &s->memory_manager_;
if (!EnsureInitialized(s)) return;
if (dict_size == 0 ||
s->params.quality == FAST_ONE_PASS_COMPRESSION_QUALITY ||
s->params.quality == FAST_TWO_PASS_COMPRESSION_QUALITY) {
return;
}
if (size > max_dict_size) {
dict += size - max_dict_size;
dict_size = max_dict_size;
}
CopyInputToRingBuffer(s, dict_size, dict);
s->last_flush_pos_ = dict_size;
s->last_processed_pos_ = dict_size;
if (dict_size > 0) {
s->prev_byte_ = dict[dict_size - 1];
}
if (dict_size > 1) {
s->prev_byte2_ = dict[dict_size - 2];
}
HasherPrependCustomDictionary(m, &s->hasher_, &s->params, dict_size, dict);
if (BROTLI_IS_OOM(m)) return;
}
/* Marks all input as processed.
Returns true if position wrapping occurs. */
static BROTLI_BOOL UpdateLastProcessedPos(BrotliEncoderState* s) {
@@ -1208,7 +1174,8 @@ static BROTLI_BOOL BrotliCompressBufferQuality10(
}
BrotliZopfliCreateCommands(block_size, block_start, max_backward_limit,
&nodes[0], dist_cache, &last_insert_len,
&commands[num_commands], &num_literals);
&params, &commands[num_commands],
&num_literals);
num_commands += path_size;
block_start += block_size;
metablock_size += block_size;

9
C/brotli/br_state.c
View File

@@ -41,6 +41,8 @@ void BrotliDecoderStateInitWithCustomAllocators(BrotliDecoderState* s,
s->memory_manager_opaque = opaque;
}
s->error_code = 0; /* BROTLI_DECODER_NO_ERROR */
BrotliInitBitReader(&s->br);
s->state = BROTLI_STATE_UNINITED;
s->substate_metablock_header = BROTLI_STATE_METABLOCK_HEADER_NONE;
@@ -81,11 +83,12 @@ void BrotliDecoderStateInitWithCustomAllocators(BrotliDecoderState* s,
s->distance_hgroup.codes = NULL;
s->distance_hgroup.htrees = NULL;
s->custom_dict = NULL;
s->custom_dict_size = 0;
s->is_last_metablock = 0;
s->is_uncompressed = 0;
s->is_metadata = 0;
s->should_wrap_ringbuffer = 0;
s->canny_ringbuffer_allocation = 1;
s->window_bits = 0;
s->max_distance = 0;
s->dist_rb[0] = 16;

2
C/brotli/common/constants.h
View File

@@ -38,6 +38,8 @@
BROTLI_MAX_NDIRECT + \
(BROTLI_MAX_DISTANCE_BITS << \
(BROTLI_MAX_NPOSTFIX + 1)))
/* Distance that is guaranteed to be representable in any stream. */
#define BROTLI_MAX_DISTANCE 0x3FFFFFC
/* 7.1. Context modes and context ID lookup for literals */
/* "context IDs for literals are in the range of 0..63" */

24
C/brotli/common/dictionary.h
View File

@@ -27,19 +27,33 @@ typedef struct BrotliDictionary {
* Dictionary consists of words with length of [4..24] bytes.
* Values at [0..3] and [25..31] indices should not be addressed.
*/
uint8_t size_bits_by_length[32];
const uint8_t size_bits_by_length[32];
/* assert(offset[i + 1] == offset[i] + (bits[i] ? (i << bits[i]) : 0)) */
uint32_t offsets_by_length[32];
const uint32_t offsets_by_length[32];
/* assert(data_size == offsets_by_length[31]) */
const size_t data_size;
/* Data array is not bound, and should obey to size_bits_by_length values.
Specified size matches default (RFC 7932) dictionary. */
/* assert(sizeof(data) == offsets_by_length[31]) */
uint8_t data[122784];
Specified size matches default (RFC 7932) dictionary. Its size is
defined by data_size */
const uint8_t* data;
} BrotliDictionary;
BROTLI_COMMON_API extern const BrotliDictionary* BrotliGetDictionary(void);
/**
* Sets dictionary data.
*
* When dictionary data is already set / present, this method is no-op.
*
* Dictionary data MUST be provided before BrotliGetDictionary is invoked.
* This method is used ONLY in multi-client environment (e.g. C + Java),
* to reduce storage by sharing single dictionary between implementations.
*/
BROTLI_COMMON_API void BrotliSetDictionaryData(const uint8_t* data);
#define BROTLI_MIN_DICTIONARY_WORD_LENGTH 4
#define BROTLI_MAX_DICTIONARY_WORD_LENGTH 24

2
C/brotli/common/version.h
View File

@@ -14,6 +14,6 @@
BrotliEncoderVersion methods. */
/* Semantic version, calculated as (MAJOR << 24) | (MINOR << 12) | PATCH */
#define BROTLI_VERSION 0x1000000
#define BROTLI_VERSION 0x1000001
#endif /* BROTLI_COMMON_VERSION_H_ */

7
C/brotli/dec/state.h
View File

@@ -172,7 +172,7 @@ struct BrotliDecoderStateStruct {
uint32_t space;
HuffmanCode table[32];
/* List of of symbol chains. */
/* List of heads of symbol chains. */
uint16_t* symbol_lists;
/* Storage from symbol_lists. */
uint16_t symbols_lists_array[BROTLI_HUFFMAN_MAX_CODE_LENGTH + 1 +
@@ -197,10 +197,6 @@ struct BrotliDecoderStateStruct {
uint32_t mtf_upper_bound;
uint32_t mtf[64 + 1];
/* For custom dictionaries */
const uint8_t* custom_dict;
int custom_dict_size;
/* less used attributes are in the end of this struct */
/* States inside function calls */
BrotliRunningMetablockHeaderState substate_metablock_header;
@@ -215,6 +211,7 @@ struct BrotliDecoderStateStruct {
unsigned int is_uncompressed : 1;
unsigned int is_metadata : 1;
unsigned int should_wrap_ringbuffer : 1;
unsigned int canny_ringbuffer_allocation : 1;
unsigned int size_nibbles : 8;
uint32_t window_bits;

61
C/brotli/decode.h
View File

@@ -84,8 +84,10 @@ typedef enum {
BROTLI_ERROR_CODE(_ERROR_FORMAT_, PADDING_1, -14) SEPARATOR \
BROTLI_ERROR_CODE(_ERROR_FORMAT_, PADDING_2, -15) SEPARATOR \
\
/* -16..-19 codes are reserved */ \
/* -16..-17 codes are reserved */ \
\
BROTLI_ERROR_CODE(_ERROR_, COMPOUND_DICTIONARY, -18) SEPARATOR \
BROTLI_ERROR_CODE(_ERROR_, DICTIONARY_NOT_SET, -19) SEPARATOR \
BROTLI_ERROR_CODE(_ERROR_, INVALID_ARGUMENTS, -20) SEPARATOR \
\
/* Memory allocation problems */ \
@@ -125,6 +127,29 @@ typedef enum {
*/
#define BROTLI_LAST_ERROR_CODE BROTLI_DECODER_ERROR_UNREACHABLE
/** Options to be used with ::BrotliDecoderSetParameter. */
typedef enum BrotliDecoderParameter {
/**
* Disable "canny" ring buffer allocation strategy.
*
* Ring buffer is allocated according to window size, despite the real size of
* the content.
*/
BROTLI_DECODER_PARAM_DISABLE_RING_BUFFER_REALLOCATION = 0
} BrotliDecoderParameter;
/**
* Sets the specified parameter to the given decoder instance.
*
* @param state decoder instance
* @param param parameter to set
* @param value new parameter value
* @returns ::BROTLI_FALSE if parameter is unrecognized, or value is invalid
* @returns ::BROTLI_TRUE if value is accepted
*/
BROTLI_DEC_API BROTLI_BOOL BrotliDecoderSetParameter(
BrotliDecoderState* state, BrotliDecoderParameter param, uint32_t value);
/**
* Creates an instance of ::BrotliDecoderState and initializes it.
*
@@ -207,9 +232,9 @@ BROTLI_DEC_API BrotliDecoderResult BrotliDecoderDecompress(
* allocation failed, arguments were invalid, etc.;
* use ::BrotliDecoderGetErrorCode to get detailed error code
* @returns ::BROTLI_DECODER_RESULT_NEEDS_MORE_INPUT decoding is blocked until
* more output space is provided
* @returns ::BROTLI_DECODER_RESULT_NEEDS_MORE_OUTPUT decoding is blocked until
* more input data is provided
* @returns ::BROTLI_DECODER_RESULT_NEEDS_MORE_OUTPUT decoding is blocked until
* more output space is provided
* @returns ::BROTLI_DECODER_RESULT_SUCCESS decoding is finished, no more
* input might be consumed and no more output will be produced
*/
@@ -217,31 +242,6 @@ BROTLI_DEC_API BrotliDecoderResult BrotliDecoderDecompressStream(
BrotliDecoderState* state, size_t* available_in, const uint8_t** next_in,
size_t* available_out, uint8_t** next_out, size_t* total_out);
/**
* Prepends LZ77 dictionary.
*
* Fills the fresh ::BrotliDecoderState with additional data corpus for LZ77
* backward references.
*
* @note Not to be confused with the static dictionary (see RFC7932 section 8).
* @warning The dictionary must exist in memory until decoding is done and
* is owned by the caller.
*
* Workflow:
* -# Allocate and initialize state with ::BrotliDecoderCreateInstance
* -# Invoke ::BrotliDecoderSetCustomDictionary
* -# Use ::BrotliDecoderDecompressStream
* -# Clean up and free state with ::BrotliDecoderDestroyInstance
*
* @param state decoder instance
* @param size length of @p dict; should be less or equal to 2^24 (16MiB),
* otherwise the dictionary will be ignored
* @param dict "dictionary"; @b MUST be the same as used during compression
*/
BROTLI_DEC_API void BrotliDecoderSetCustomDictionary(
BrotliDecoderState* state, size_t size,
const uint8_t dict[BROTLI_ARRAY_PARAM(size)]);
/**
* Checks if decoder has more output.
*
@@ -303,7 +303,8 @@ BROTLI_DEC_API BROTLI_BOOL BrotliDecoderIsUsed(const BrotliDecoderState* state);
* the input and produced all of the output
* @returns ::BROTLI_FALSE otherwise
*/
BROTLI_BOOL BrotliDecoderIsFinished(const BrotliDecoderState* state);
BROTLI_DEC_API BROTLI_BOOL BrotliDecoderIsFinished(
const BrotliDecoderState* state);
/**
* Acquires a detailed error code.
@@ -316,7 +317,7 @@ BROTLI_BOOL BrotliDecoderIsFinished(const BrotliDecoderState* state);
* @param state decoder instance
* @returns last saved error code
*/
BrotliDecoderErrorCode BrotliDecoderGetErrorCode(
BROTLI_DEC_API BrotliDecoderErrorCode BrotliDecoderGetErrorCode(
const BrotliDecoderState* state);
/**

13
C/brotli/enc/backward_references_hq.h
View File

@@ -83,14 +83,11 @@ BROTLI_INTERNAL size_t BrotliZopfliComputeShortestPath(
const BrotliEncoderParams* params, const size_t max_backward_limit,
const int* dist_cache, HasherHandle hasher, ZopfliNode* nodes);
BROTLI_INTERNAL void BrotliZopfliCreateCommands(const size_t num_bytes,
const size_t block_start,
const size_t max_backward_limit,
const ZopfliNode* nodes,
int* dist_cache,
size_t* last_insert_len,
Command* commands,
size_t* num_literals);
BROTLI_INTERNAL void BrotliZopfliCreateCommands(
const size_t num_bytes, const size_t block_start,
const size_t max_backward_limit, const ZopfliNode* nodes,
int* dist_cache, size_t* last_insert_len, const BrotliEncoderParams* params,
Command* commands, size_t* num_literals);
#if defined(__cplusplus) || defined(c_plusplus)
} /* extern "C" */

50
C/brotli/enc/backward_references_inc.h
View File

@@ -5,11 +5,11 @@
See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
*/
/* template parameters: FN */
/* template parameters: EXPORT_FN, FN */
static BROTLI_NOINLINE void FN(CreateBackwardReferences)(
const BrotliDictionary* dictionary, const uint16_t* dictionary_hash,
size_t num_bytes, size_t position,
static BROTLI_NOINLINE void EXPORT_FN(CreateBackwardReferences)(
const BrotliDictionary* dictionary,
const uint16_t* dictionary_hash, size_t num_bytes, size_t position,
const uint8_t* ringbuffer, size_t ringbuffer_mask,
const BrotliEncoderParams* params, HasherHandle hasher, int* dist_cache,
size_t* last_insert_len, Command* commands, size_t* num_commands,
@@ -27,6 +27,7 @@ static BROTLI_NOINLINE void FN(CreateBackwardReferences)(
const size_t random_heuristics_window_size =
LiteralSpreeLengthForSparseSearch(params);
size_t apply_random_heuristics = position + random_heuristics_window_size;
const size_t gap = 0;
/* Minimum score to accept a backward reference. */
const score_t kMinScore = BROTLI_SCORE_BASE + 100;
@@ -38,29 +39,29 @@ static BROTLI_NOINLINE void FN(CreateBackwardReferences)(
size_t max_distance = BROTLI_MIN(size_t, position, max_backward_limit);
HasherSearchResult sr;
sr.len = 0;
sr.len_x_code = 0;
sr.len_code_delta = 0;
sr.distance = 0;
sr.score = kMinScore;
if (FN(FindLongestMatch)(hasher, dictionary, dictionary_hash,
ringbuffer, ringbuffer_mask, dist_cache,
position, max_length, max_distance, &sr)) {
FN(FindLongestMatch)(hasher, dictionary, dictionary_hash, ringbuffer,
ringbuffer_mask, dist_cache, position,
max_length, max_distance, gap, &sr);
if (sr.score > kMinScore) {
/* Found a match. Let's look for something even better ahead. */
int delayed_backward_references_in_row = 0;
--max_length;
for (;; --max_length) {
const score_t cost_diff_lazy = 175;
BROTLI_BOOL is_match_found;
HasherSearchResult sr2;
sr2.len = params->quality < MIN_QUALITY_FOR_EXTENSIVE_REFERENCE_SEARCH ?
BROTLI_MIN(size_t, sr.len - 1, max_length) : 0;
sr2.len_x_code = 0;
sr2.len_code_delta = 0;
sr2.distance = 0;
sr2.score = kMinScore;
max_distance = BROTLI_MIN(size_t, position + 1, max_backward_limit);
is_match_found = FN(FindLongestMatch)(hasher, dictionary,
dictionary_hash, ringbuffer, ringbuffer_mask, dist_cache,
position + 1, max_length, max_distance, &sr2);
if (is_match_found && sr2.score >= sr.score + cost_diff_lazy) {
FN(FindLongestMatch)(hasher, dictionary, dictionary_hash, ringbuffer,
ringbuffer_mask, dist_cache, position + 1,
max_length, max_distance, gap, &sr2);
if (sr2.score >= sr.score + cost_diff_lazy) {
/* Ok, let's just write one byte for now and start a match from the
next byte. */
++position;
@@ -80,24 +81,33 @@ static BROTLI_NOINLINE void FN(CreateBackwardReferences)(
/* The first 16 codes are special short-codes,
and the minimum offset is 1. */
size_t distance_code =
ComputeDistanceCode(sr.distance, max_distance, dist_cache);
if (sr.distance <= max_distance && distance_code > 0) {
ComputeDistanceCode(sr.distance, max_distance + gap, dist_cache);
if ((sr.distance <= (max_distance + gap)) && distance_code > 0) {
dist_cache[3] = dist_cache[2];
dist_cache[2] = dist_cache[1];
dist_cache[1] = dist_cache[0];
dist_cache[0] = (int)sr.distance;
FN(PrepareDistanceCache)(hasher, dist_cache);
}
InitCommand(commands++, insert_length, sr.len, sr.len ^ sr.len_x_code,
InitCommand(commands++, insert_length, sr.len, sr.len_code_delta,
distance_code);
}
*num_literals += insert_length;
insert_length = 0;
/* Put the hash keys into the table, if there are enough bytes left.
Depending on the hasher implementation, it can push all positions
in the given range or only a subset of them. */
FN(StoreRange)(hasher, ringbuffer, ringbuffer_mask, position + 2,
BROTLI_MIN(size_t, position + sr.len, store_end));
in the given range or only a subset of them.
Avoid hash poisoning with RLE data. */
{
size_t range_start = position + 2;
size_t range_end = BROTLI_MIN(size_t, position + sr.len, store_end);
if (sr.distance < (sr.len >> 2)) {
range_start = BROTLI_MIN(size_t, range_end, BROTLI_MAX(size_t,
range_start, position + sr.len - (sr.distance << 2)));
}
FN(StoreRange)(hasher, ringbuffer, ringbuffer_mask, range_start,
range_end);
}
position += sr.len;
} else {
++insert_length;

7
C/brotli/enc/block_splitter_inc.h
View File

@@ -13,7 +13,7 @@ static void FN(InitialEntropyCodes)(const DataType* data, size_t length,
size_t stride,
size_t num_histograms,
HistogramType* histograms) {
unsigned int seed = 7;
uint32_t seed = 7;
size_t block_length = length / num_histograms;
size_t i;
FN(ClearHistograms)(histograms, num_histograms);
@@ -29,14 +29,13 @@ static void FN(InitialEntropyCodes)(const DataType* data, size_t length,
}
}
static void FN(RandomSample)(unsigned int* seed,
static void FN(RandomSample)(uint32_t* seed,
const DataType* data,
size_t length,
size_t stride,
HistogramType* sample) {
size_t pos = 0;
if (stride >= length) {
pos = 0;
stride = length;
} else {
pos = MyRand(seed) % (length - stride + 1);
@@ -50,7 +49,7 @@ static void FN(RefineEntropyCodes)(const DataType* data, size_t length,
HistogramType* histograms) {
size_t iters =
kIterMulForRefining * length / stride + kMinItersForRefining;
unsigned int seed = 7;
uint32_t seed = 7;
size_t iter;
iters = ((iters + num_histograms - 1) / num_histograms) * num_histograms;
for (iter = 0; iter < iters; ++iter) {

4
C/brotli/enc/brotli_bit_stream.h
View File

@@ -96,10 +96,6 @@ BROTLI_INTERNAL void BrotliStoreUncompressedMetaBlock(
BROTLI_BOOL is_final_block, const uint8_t* input, size_t position,
size_t mask, size_t len, size_t* storage_ix, uint8_t* storage);
/* Stores an empty metadata meta-block and syncs to a byte boundary. */
BROTLI_INTERNAL void BrotliStoreSyncMetaBlock(size_t* storage_ix,
uint8_t* storage);
#if defined(__cplusplus) || defined(c_plusplus)
} /* extern "C" */
#endif

2
C/brotli/enc/cluster_inc.h
View File

@@ -17,6 +17,8 @@ BROTLI_INTERNAL void FN(BrotliCompareAndPushToQueue)(
size_t* num_pairs) CODE({
BROTLI_BOOL is_good_pair = BROTLI_FALSE;
HistogramPair p;
p.idx1 = p.idx2 = 0;
p.cost_diff = p.cost_combo = 0;
if (idx1 == idx2) {
return;
}

13
C/brotli/enc/command.h
View File

@@ -114,17 +114,19 @@ typedef struct Command {
/* distance_code is e.g. 0 for same-as-last short code, or 16 for offset 1. */
static BROTLI_INLINE void InitCommand(Command* self, size_t insertlen,
size_t copylen, size_t copylen_code, size_t distance_code) {
size_t copylen, int copylen_code_delta, size_t distance_code) {
/* Don't rely on signed int representation, use honest casts. */
uint32_t delta = (uint8_t)((int8_t)copylen_code_delta);
self->insert_len_ = (uint32_t)insertlen;
self->copy_len_ = (uint32_t)(copylen | ((copylen_code ^ copylen) << 24));
self->copy_len_ = (uint32_t)(copylen | (delta << 24));
/* The distance prefix and extra bits are stored in this Command as if
npostfix and ndirect were 0, they are only recomputed later after the
clustering if needed. */
PrefixEncodeCopyDistance(
distance_code, 0, 0, &self->dist_prefix_, &self->dist_extra_);
GetLengthCode(
insertlen, copylen_code, TO_BROTLI_BOOL(self->dist_prefix_ == 0),
&self->cmd_prefix_);
insertlen, (size_t)((int)copylen + copylen_code_delta),
TO_BROTLI_BOOL(self->dist_prefix_ == 0), &self->cmd_prefix_);
}
static BROTLI_INLINE void InitInsertCommand(Command* self, size_t insertlen) {
@@ -167,7 +169,8 @@ static BROTLI_INLINE uint32_t CommandCopyLen(const Command* self) {
}
static BROTLI_INLINE uint32_t CommandCopyLenCode(const Command* self) {
return (self->copy_len_ & 0xFFFFFF) ^ (self->copy_len_ >> 24);
int32_t delta = (int8_t)((uint8_t)(self->copy_len_ >> 24));
return (uint32_t)((int32_t)(self->copy_len_ & 0xFFFFFF) + delta);
}
#if defined(__cplusplus) || defined(c_plusplus)

10
C/brotli/enc/find_match_length.h
View File

@@ -17,7 +17,7 @@ extern "C" {
#endif
/* Separate implementation for little-endian 64-bit targets, for speed. */
#if defined(__GNUC__) && defined(_LP64) && defined(IS_LITTLE_ENDIAN)
#if defined(__GNUC__) && defined(_LP64) && defined(BROTLI_LITTLE_ENDIAN)
static BROTLI_INLINE size_t FindMatchLengthWithLimit(const uint8_t* s1,
const uint8_t* s2,
@@ -25,13 +25,13 @@ static BROTLI_INLINE size_t FindMatchLengthWithLimit(const uint8_t* s1,
size_t matched = 0;
size_t limit2 = (limit >> 3) + 1; /* + 1 is for pre-decrement in while */
while (BROTLI_PREDICT_TRUE(--limit2)) {
if (BROTLI_PREDICT_FALSE(BROTLI_UNALIGNED_LOAD64(s2) ==
BROTLI_UNALIGNED_LOAD64(s1 + matched))) {
if (BROTLI_PREDICT_FALSE(BROTLI_UNALIGNED_LOAD64LE(s2) ==
BROTLI_UNALIGNED_LOAD64LE(s1 + matched))) {
s2 += 8;
matched += 8;
} else {
uint64_t x =
BROTLI_UNALIGNED_LOAD64(s2) ^ BROTLI_UNALIGNED_LOAD64(s1 + matched);
uint64_t x = BROTLI_UNALIGNED_LOAD64LE(s2) ^
BROTLI_UNALIGNED_LOAD64LE(s1 + matched);
size_t matching_bits = (size_t)__builtin_ctzll(x);
matched += matching_bits >> 3;
return matched;

38
C/brotli/enc/hash.h
View File

@@ -62,9 +62,9 @@ static const uint64_t kCutoffTransforms =
typedef struct HasherSearchResult {
size_t len;
size_t len_x_code; /* == len ^ len_code */
size_t distance;
score_t score;
int len_code_delta; /* == len_code - len */
} HasherSearchResult;
/* kHashMul32 multiplier has these properties:
@@ -173,27 +173,29 @@ static BROTLI_INLINE BROTLI_BOOL TestStaticDictionaryItem(
backward = max_backward + dist + 1 +
(transform_id << dictionary->size_bits_by_length[len]);
}
if (backward >= BROTLI_MAX_DISTANCE) {
return BROTLI_FALSE;
}
score = BackwardReferenceScore(matchlen, backward);
if (score < out->score) {
return BROTLI_FALSE;
}
out->len = matchlen;
out->len_x_code = len ^ matchlen;
out->len_code_delta = (int)len - (int)matchlen;
out->distance = backward;
out->score = score;
return BROTLI_TRUE;
}
static BROTLI_INLINE BROTLI_BOOL SearchInStaticDictionary(
static BROTLI_INLINE void SearchInStaticDictionary(
const BrotliDictionary* dictionary, const uint16_t* dictionary_hash,
HasherHandle handle, const uint8_t* data, size_t max_length,
size_t max_backward, HasherSearchResult* out, BROTLI_BOOL shallow) {
size_t key;
size_t i;
BROTLI_BOOL is_match_found = BROTLI_FALSE;
HasherCommon* self = GetHasherCommon(handle);
if (self->dict_num_matches < (self->dict_num_lookups >> 7)) {
return BROTLI_FALSE;
return;
}
key = Hash14(data) << 1;
for (i = 0; i < (shallow ? 1u : 2u); ++i, ++key) {
@@ -204,11 +206,9 @@ static BROTLI_INLINE BROTLI_BOOL SearchInStaticDictionary(
dictionary, item, data, max_length, max_backward, out);
if (item_matches) {
self->dict_num_matches++;
is_match_found = BROTLI_TRUE;
}
}
}
return is_match_found;
}
typedef struct BackwardMatch {
@@ -420,30 +420,6 @@ static BROTLI_INLINE void HasherSetup(MemoryManager* m, HasherHandle* handle,
}
}
/* Custom LZ77 window. */
static BROTLI_INLINE void HasherPrependCustomDictionary(
MemoryManager* m, HasherHandle* handle, BrotliEncoderParams* params,
const size_t size, const uint8_t* dict) {
size_t overlap;
size_t i;
HasherHandle self;
HasherSetup(m, handle, params, dict, 0, size, BROTLI_FALSE);
if (BROTLI_IS_OOM(m)) return;
self = *handle;
switch (GetHasherCommon(self)->params.type) {
#define PREPEND_(N) \
case N: \
overlap = (StoreLookaheadH ## N()) - 1; \
for (i = 0; i + overlap < size; i++) { \
StoreH ## N(self, dict, ~(size_t)0, i); \
} \
break;
FOR_ALL_HASHERS(PREPEND_)
#undef PREPEND_
default: break;
}
}
static BROTLI_INLINE void InitOrStitchToPreviousBlock(
MemoryManager* m, HasherHandle* handle, const uint8_t* data, size_t mask,
BrotliEncoderParams* params, size_t position, size_t input_size,

19
C/brotli/enc/hash_forgetful_chain_inc.h
View File

@@ -152,24 +152,24 @@ static BROTLI_INLINE void FN(PrepareDistanceCache)(
Does not look for matches longer than max_length.
Does not look for matches further away than max_backward.
Writes the best match into |out|.
Returns 1 when match is found, otherwise 0. */
static BROTLI_INLINE BROTLI_BOOL FN(FindLongestMatch)(HasherHandle handle,
|out|->score is updated only if a better match is found. */
static BROTLI_INLINE void FN(FindLongestMatch)(HasherHandle handle,
const BrotliDictionary* dictionary, const uint16_t* dictionary_hash,
const uint8_t* BROTLI_RESTRICT data, const size_t ring_buffer_mask,
const int* BROTLI_RESTRICT distance_cache,
const size_t cur_ix, const size_t max_length, const size_t max_backward,
HasherSearchResult* BROTLI_RESTRICT out) {
const size_t gap, HasherSearchResult* BROTLI_RESTRICT out) {
HashForgetfulChain* self = FN(Self)(handle);
const size_t cur_ix_masked = cur_ix & ring_buffer_mask;
BROTLI_BOOL is_match_found = BROTLI_FALSE;
/* Don't accept a short copy from far away. */
score_t min_score = out->score;
score_t best_score = out->score;
size_t best_len = out->len;
size_t i;
const size_t key = FN(HashBytes)(&data[cur_ix_masked]);
const uint8_t tiny_hash = (uint8_t)(key);
out->len = 0;
out->len_x_code = 0;
out->len_code_delta = 0;
/* Try last distance first. */
for (i = 0; i < NUM_LAST_DISTANCES_TO_CHECK; ++i) {
const size_t backward = (size_t)distance_cache[i];
@@ -194,7 +194,6 @@ static BROTLI_INLINE BROTLI_BOOL FN(FindLongestMatch)(HasherHandle handle,
out->len = best_len;
out->distance = backward;
out->score = best_score;
is_match_found = BROTLI_TRUE;
}
}
}
@@ -234,19 +233,17 @@ static BROTLI_INLINE BROTLI_BOOL FN(FindLongestMatch)(HasherHandle handle,
out->len = best_len;
out->distance = backward;
out->score = best_score;
is_match_found = BROTLI_TRUE;
}
}
}
}
FN(Store)(handle, data, ring_buffer_mask, cur_ix);
}
if (!is_match_found) {
is_match_found = SearchInStaticDictionary(dictionary, dictionary_hash,
handle, &data[cur_ix_masked], max_length, max_backward, out,
if (out->score == min_score) {
SearchInStaticDictionary(dictionary, dictionary_hash,
handle, &data[cur_ix_masked], max_length, max_backward + gap, out,
BROTLI_FALSE);
}
return is_match_found;
}
#undef BANK_SIZE

21
C/brotli/enc/hash_longest_match64_inc.h
View File

@@ -23,7 +23,7 @@ static BROTLI_INLINE size_t FN(StoreLookahead)(void) { return 8; }
static BROTLI_INLINE uint32_t FN(HashBytes)(const uint8_t *data,
const uint64_t mask,
const int shift) {
const uint64_t h = (BROTLI_UNALIGNED_LOAD64(data) & mask) * kHashMul64Long;
const uint64_t h = (BROTLI_UNALIGNED_LOAD64LE(data) & mask) * kHashMul64Long;
/* The higher bits contain more mixture from the multiplication,
so we take our results from there. */
return (uint32_t)(h >> shift);
@@ -156,25 +156,25 @@ static BROTLI_INLINE void FN(PrepareDistanceCache)(
Does not look for matches longer than max_length.
Does not look for matches further away than max_backward.
Writes the best match into |out|.
Returns true when match is found, otherwise false. */
static BROTLI_INLINE BROTLI_BOOL FN(FindLongestMatch)(HasherHandle handle,
|out|->score is updated only if a better match is found. */
static BROTLI_INLINE void FN(FindLongestMatch)(HasherHandle handle,
const BrotliDictionary* dictionary, const uint16_t* dictionary_hash,
const uint8_t* BROTLI_RESTRICT data, const size_t ring_buffer_mask,
const int* BROTLI_RESTRICT distance_cache, const size_t cur_ix,
const size_t max_length, const size_t max_backward,
const size_t max_length, const size_t max_backward, const size_t gap,
HasherSearchResult* BROTLI_RESTRICT out) {
HasherCommon* common = GetHasherCommon(handle);
HashLongestMatch* self = FN(Self)(handle);
uint16_t* num = FN(Num)(self);
uint32_t* buckets = FN(Buckets)(self);
const size_t cur_ix_masked = cur_ix & ring_buffer_mask;
BROTLI_BOOL is_match_found = BROTLI_FALSE;
/* Don't accept a short copy from far away. */
score_t min_score = out->score;
score_t best_score = out->score;
size_t best_len = out->len;
size_t i;
out->len = 0;
out->len_x_code = 0;
out->len_code_delta = 0;
/* Try last distance first. */
for (i = 0; i < (size_t)common->params.num_last_distances_to_check; ++i) {
const size_t backward = (size_t)distance_cache[i];
@@ -209,7 +209,6 @@ static BROTLI_INLINE BROTLI_BOOL FN(FindLongestMatch)(HasherHandle handle,
out->len = best_len;
out->distance = backward;
out->score = best_score;
is_match_found = BROTLI_TRUE;
}
}
}
@@ -250,7 +249,6 @@ static BROTLI_INLINE BROTLI_BOOL FN(FindLongestMatch)(HasherHandle handle,
out->len = best_len;
out->distance = backward;
out->score = best_score;
is_match_found = BROTLI_TRUE;
}
}
}
@@ -258,12 +256,11 @@ static BROTLI_INLINE BROTLI_BOOL FN(FindLongestMatch)(HasherHandle handle,
bucket[num[key] & self->block_mask_] = (uint32_t)cur_ix;
++num[key];
}
if (!is_match_found) {
is_match_found = SearchInStaticDictionary(dictionary, dictionary_hash,
handle, &data[cur_ix_masked], max_length, max_backward, out,
if (min_score == out->score) {
SearchInStaticDictionary(dictionary, dictionary_hash,
handle, &data[cur_ix_masked], max_length, max_backward + gap, out,
BROTLI_FALSE);
}
return is_match_found;
}
#undef HashLongestMatch

19
C/brotli/enc/hash_longest_match_inc.h
View File

@@ -149,25 +149,25 @@ static BROTLI_INLINE void FN(PrepareDistanceCache)(
Does not look for matches longer than max_length.
Does not look for matches further away than max_backward.
Writes the best match into |out|.
Returns true when match is found, otherwise false. */
static BROTLI_INLINE BROTLI_BOOL FN(FindLongestMatch)(HasherHandle handle,
|out|->score is updated only if a better match is found. */
static BROTLI_INLINE void FN(FindLongestMatch)(HasherHandle handle,
const BrotliDictionary* dictionary, const uint16_t* dictionary_hash,
const uint8_t* BROTLI_RESTRICT data, const size_t ring_buffer_mask,
const int* BROTLI_RESTRICT distance_cache, const size_t cur_ix,
const size_t max_length, const size_t max_backward,
const size_t max_length, const size_t max_backward, const size_t gap,
HasherSearchResult* BROTLI_RESTRICT out) {
HasherCommon* common = GetHasherCommon(handle);
HashLongestMatch* self = FN(Self)(handle);
uint16_t* num = FN(Num)(self);
uint32_t* buckets = FN(Buckets)(self);
const size_t cur_ix_masked = cur_ix & ring_buffer_mask;
BROTLI_BOOL is_match_found = BROTLI_FALSE;
/* Don't accept a short copy from far away. */
score_t min_score = out->score;
score_t best_score = out->score;
size_t best_len = out->len;
size_t i;
out->len = 0;
out->len_x_code = 0;
out->len_code_delta = 0;
/* Try last distance first. */
for (i = 0; i < (size_t)common->params.num_last_distances_to_check; ++i) {
const size_t backward = (size_t)distance_cache[i];
@@ -202,7 +202,6 @@ static BROTLI_INLINE BROTLI_BOOL FN(FindLongestMatch)(HasherHandle handle,
out->len = best_len;
out->distance = backward;
out->score = best_score;
is_match_found = BROTLI_TRUE;
}
}
}
@@ -242,7 +241,6 @@ static BROTLI_INLINE BROTLI_BOOL FN(FindLongestMatch)(HasherHandle handle,
out->len = best_len;
out->distance = backward;
out->score = best_score;
is_match_found = BROTLI_TRUE;
}
}
}
@@ -250,12 +248,11 @@ static BROTLI_INLINE BROTLI_BOOL FN(FindLongestMatch)(HasherHandle handle,
bucket[num[key] & self->block_mask_] = (uint32_t)cur_ix;
++num[key];
}
if (!is_match_found) {
is_match_found = SearchInStaticDictionary(dictionary, dictionary_hash,
handle, &data[cur_ix_masked], max_length, max_backward, out,
if (min_score == out->score) {
SearchInStaticDictionary(dictionary, dictionary_hash,
handle, &data[cur_ix_masked], max_length, max_backward + gap, out,
BROTLI_FALSE);
}
return is_match_found;
}
#undef HashLongestMatch

58
C/brotli/enc/hash_longest_match_quickly_inc.h
View File

@@ -22,7 +22,7 @@ static BROTLI_INLINE size_t FN(StoreLookahead)(void) { return 8; }
the address in. The HashLongestMatch and HashLongestMatchQuickly
classes have separate, different implementations of hashing. */
static uint32_t FN(HashBytes)(const uint8_t* data) {
const uint64_t h = ((BROTLI_UNALIGNED_LOAD64(data) << (64 - 8 * HASH_LEN)) *
const uint64_t h = ((BROTLI_UNALIGNED_LOAD64LE(data) << (64 - 8 * HASH_LEN)) *
kHashMul64);
/* The higher bits contain more mixture from the multiplication,
so we take our results from there. */
@@ -123,24 +123,24 @@ static BROTLI_INLINE void FN(PrepareDistanceCache)(
Does not look for matches longer than max_length.
Does not look for matches further away than max_backward.
Writes the best match into |out|.
Returns true if match is found, otherwise false. */
static BROTLI_INLINE BROTLI_BOOL FN(FindLongestMatch)(
|out|->score is updated only if a better match is found. */
static BROTLI_INLINE void FN(FindLongestMatch)(
HasherHandle handle, const BrotliDictionary* dictionary,
const uint16_t* dictionary_hash, const uint8_t* BROTLI_RESTRICT data,
const size_t ring_buffer_mask, const int* BROTLI_RESTRICT distance_cache,
const size_t cur_ix, const size_t max_length, const size_t max_backward,
HasherSearchResult* BROTLI_RESTRICT out) {
const size_t gap, HasherSearchResult* BROTLI_RESTRICT out) {
HashLongestMatchQuickly* self = FN(Self)(handle);
const size_t best_len_in = out->len;
const size_t cur_ix_masked = cur_ix & ring_buffer_mask;
const uint32_t key = FN(HashBytes)(&data[cur_ix_masked]);
int compare_char = data[cur_ix_masked + best_len_in];
score_t min_score = out->score;
score_t best_score = out->score;
size_t best_len = best_len_in;
size_t cached_backward = (size_t)distance_cache[0];
size_t prev_ix = cur_ix - cached_backward;
BROTLI_BOOL is_match_found = BROTLI_FALSE;
out->len_x_code = 0;
out->len_code_delta = 0;
if (prev_ix < cur_ix) {
prev_ix &= (uint32_t)ring_buffer_mask;
if (compare_char == data[prev_ix + best_len]) {
@@ -148,17 +148,18 @@ static BROTLI_INLINE BROTLI_BOOL FN(FindLongestMatch)(
&data[cur_ix_masked],
max_length);
if (len >= 4) {
best_score = BackwardReferenceScoreUsingLastDistance(len);
best_len = len;
out->len = len;
out->distance = cached_backward;
out->score = best_score;
compare_char = data[cur_ix_masked + best_len];
if (BUCKET_SWEEP == 1) {
self->buckets_[key] = (uint32_t)cur_ix;
return BROTLI_TRUE;
} else {
is_match_found = BROTLI_TRUE;
const score_t score = BackwardReferenceScoreUsingLastDistance(len);
if (best_score < score) {
best_score = score;
best_len = len;
out->len = len;
out->distance = cached_backward;
out->score = best_score;
compare_char = data[cur_ix_masked + best_len];
if (BUCKET_SWEEP == 1) {
self->buckets_[key] = (uint32_t)cur_ix;
return;
}
}
}
}
@@ -172,19 +173,22 @@ static BROTLI_INLINE BROTLI_BOOL FN(FindLongestMatch)(
backward = cur_ix - prev_ix;
prev_ix &= (uint32_t)ring_buffer_mask;
if (compare_char != data[prev_ix + best_len_in]) {
return BROTLI_FALSE;
return;
}
if (BROTLI_PREDICT_FALSE(backward == 0 || backward > max_backward)) {
return BROTLI_FALSE;
return;
}
len = FindMatchLengthWithLimit(&data[prev_ix],
&data[cur_ix_masked],
max_length);
if (len >= 4) {
out->len = len;
out->distance = backward;
out->score = BackwardReferenceScore(len, backward);
return BROTLI_TRUE;
const score_t score = BackwardReferenceScore(len, backward);
if (best_score < score) {
out->len = len;
out->distance = backward;
out->score = score;
return;
}
}
} else {
uint32_t *bucket = self->buckets_ + key;
@@ -212,18 +216,16 @@ static BROTLI_INLINE BROTLI_BOOL FN(FindLongestMatch)(
out->distance = backward;
out->score = score;
compare_char = data[cur_ix_masked + best_len];
is_match_found = BROTLI_TRUE;
}
}
}
}
if (USE_DICTIONARY && !is_match_found) {
is_match_found = SearchInStaticDictionary(dictionary, dictionary_hash,
handle, &data[cur_ix_masked], max_length, max_backward, out,
if (USE_DICTIONARY && min_score == out->score) {
SearchInStaticDictionary(dictionary, dictionary_hash,
handle, &data[cur_ix_masked], max_length, max_backward + gap, out,
BROTLI_TRUE);
}
self->buckets_[key + ((cur_ix >> 3) % BUCKET_SWEEP)] = (uint32_t)cur_ix;
return is_match_found;
}
#undef HASH_MAP_SIZE

14
C/brotli/enc/hash_to_binary_tree_inc.h
View File

@@ -19,8 +19,10 @@
#define BUCKET_SIZE (1 << BUCKET_BITS)
static size_t FN(HashTypeLength)(void) { return 4; }
static size_t FN(StoreLookahead)(void) { return MAX_TREE_COMP_LENGTH; }
static BROTLI_INLINE size_t FN(HashTypeLength)(void) { return 4; }
static BROTLI_INLINE size_t FN(StoreLookahead)(void) {
return MAX_TREE_COMP_LENGTH;
}
static uint32_t FN(HashBytes)(const uint8_t *data) {
uint32_t h = BROTLI_UNALIGNED_LOAD32(data) * kHashMul32;
@@ -199,7 +201,7 @@ static BROTLI_INLINE BackwardMatch* FN(StoreAndFindMatches)(
static BROTLI_INLINE size_t FN(FindAllMatches)(HasherHandle handle,
const BrotliDictionary* dictionary, const uint8_t* data,
const size_t ring_buffer_mask, const size_t cur_ix,
const size_t max_length, const size_t max_backward,
const size_t max_length, const size_t max_backward, const size_t gap,
const BrotliEncoderParams* params, BackwardMatch* matches) {
BackwardMatch* const orig_matches = matches;
const size_t cur_ix_masked = cur_ix & ring_buffer_mask;
@@ -248,8 +250,10 @@ static BROTLI_INLINE size_t FN(FindAllMatches)(HasherHandle handle,
for (l = minlen; l <= maxlen; ++l) {
uint32_t dict_id = dict_matches[l];
if (dict_id < kInvalidMatch) {
InitDictionaryBackwardMatch(matches++,
max_backward + (dict_id >> 5) + 1, l, dict_id & 31);
size_t distance = max_backward + gap + (dict_id >> 5) + 1;
if (distance < BROTLI_MAX_DISTANCE) {
InitDictionaryBackwardMatch(matches++, distance, l, dict_id & 31);
}
}
}
}

2
C/brotli/enc/memory.h
View File

@@ -40,7 +40,7 @@ BROTLI_INTERNAL void BrotliInitMemoryManager(
BROTLI_INTERNAL void* BrotliAllocate(MemoryManager* m, size_t n);
#define BROTLI_ALLOC(M, T, N) \
((N) ? ((T*)BrotliAllocate((M), (N) * sizeof(T))) : NULL)
((N) > 0 ? ((T*)BrotliAllocate((M), (N) * sizeof(T))) : NULL)
BROTLI_INTERNAL void BrotliFree(MemoryManager* m, void* p);
#define BROTLI_FREE(M, P) { \

64
C/brotli/enc/port.h
View File

@@ -26,36 +26,37 @@
#define __LITTLE_ENDIAN LITTLE_ENDIAN
#endif
/* define the macro IS_LITTLE_ENDIAN
/* define the macro BROTLI_LITTLE_ENDIAN
using the above endian definitions from endian.h if
endian.h was included */
#ifdef __BYTE_ORDER
#if __BYTE_ORDER == __LITTLE_ENDIAN
#define IS_LITTLE_ENDIAN
#define BROTLI_LITTLE_ENDIAN
#endif
#else
#if defined(__LITTLE_ENDIAN__)
#define IS_LITTLE_ENDIAN
#define BROTLI_LITTLE_ENDIAN
#endif
#endif /* __BYTE_ORDER */
#if defined(__BYTE_ORDER__) && (__BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__)
#define IS_LITTLE_ENDIAN
#define BROTLI_LITTLE_ENDIAN
#endif
/* Enable little-endian optimization for x64 architecture on Windows. */
#if (defined(_WIN32) || defined(_WIN64)) && defined(_M_X64)
#define IS_LITTLE_ENDIAN
#define BROTLI_LITTLE_ENDIAN
#endif
/* Portable handling of unaligned loads, stores, and copies.
On some platforms, like ARM, the copy functions can be more efficient
then a load and a store. */
#if defined(ARCH_PIII) || \
defined(ARCH_ATHLON) || defined(ARCH_K8) || defined(_ARCH_PPC)
#if defined(BROTLI_LITTLE_ENDIAN) && (\
defined(ARCH_PIII) || defined(ARCH_ATHLON) || \
defined(ARCH_K8) || defined(_ARCH_PPC))
/* x86 and x86-64 can perform unaligned loads/stores directly;
modern PowerPC hardware can also do unaligned integer loads and stores;
@@ -63,14 +64,12 @@
*/
#define BROTLI_UNALIGNED_LOAD32(_p) (*(const uint32_t *)(_p))
#define BROTLI_UNALIGNED_LOAD64(_p) (*(const uint64_t *)(_p))
#define BROTLI_UNALIGNED_LOAD64LE(_p) (*(const uint64_t *)(_p))
#define BROTLI_UNALIGNED_STORE32(_p, _val) \
(*(uint32_t *)(_p) = (_val))
#define BROTLI_UNALIGNED_STORE64(_p, _val) \
#define BROTLI_UNALIGNED_STORE64LE(_p, _val) \
(*(uint64_t *)(_p) = (_val))
#elif defined(__arm__) && \
#elif defined(BROTLI_LITTLE_ENDIAN) && defined(__arm__) && \
!defined(__ARM_ARCH_5__) && \
!defined(__ARM_ARCH_5T__) && \
!defined(__ARM_ARCH_5TE__) && \
@@ -88,16 +87,14 @@
slowly (trip through kernel mode). */
#define BROTLI_UNALIGNED_LOAD32(_p) (*(const uint32_t *)(_p))
#define BROTLI_UNALIGNED_STORE32(_p, _val) \
(*(uint32_t *)(_p) = (_val))
static BROTLI_INLINE uint64_t BROTLI_UNALIGNED_LOAD64(const void *p) {
static BROTLI_INLINE uint64_t BROTLI_UNALIGNED_LOAD64LE(const void *p) {
uint64_t t;
memcpy(&t, p, sizeof t);
return t;
}
static BROTLI_INLINE void BROTLI_UNALIGNED_STORE64(void *p, uint64_t v) {
static BROTLI_INLINE void BROTLI_UNALIGNED_STORE64LE(void *p, uint64_t v) {
memcpy(p, &v, sizeof v);
}
@@ -112,20 +109,47 @@ static BROTLI_INLINE uint32_t BROTLI_UNALIGNED_LOAD32(const void *p) {
return t;
}
static BROTLI_INLINE uint64_t BROTLI_UNALIGNED_LOAD64(const void *p) {
#if defined(BROTLI_LITTLE_ENDIAN)
static BROTLI_INLINE uint64_t BROTLI_UNALIGNED_LOAD64LE(const void *p) {
uint64_t t;
memcpy(&t, p, sizeof t);
return t;
}
static BROTLI_INLINE void BROTLI_UNALIGNED_STORE32(void *p, uint32_t v) {
static BROTLI_INLINE void BROTLI_UNALIGNED_STORE64LE(void *p, uint64_t v) {
memcpy(p, &v, sizeof v);
}
static BROTLI_INLINE void BROTLI_UNALIGNED_STORE64(void *p, uint64_t v) {
memcpy(p, &v, sizeof v);
#else /* BROTLI_LITTLE_ENDIAN */
static BROTLI_INLINE uint64_t BROTLI_UNALIGNED_LOAD64LE(const void *p) {
const uint8_t* in = (const uint8_t*)p;
uint64_t value = (uint64_t)(in[0]);
value |= (uint64_t)(in[1]) << 8;
value |= (uint64_t)(in[2]) << 16;
value |= (uint64_t)(in[3]) << 24;
value |= (uint64_t)(in[4]) << 32;
value |= (uint64_t)(in[5]) << 40;
value |= (uint64_t)(in[6]) << 48;
value |= (uint64_t)(in[7]) << 56;
return value;
}
static BROTLI_INLINE void BROTLI_UNALIGNED_STORE64LE(void *p, uint64_t v) {
uint8_t* out = (uint8_t*)p;
out[0] = (uint8_t)v;
out[1] = (uint8_t)(v >> 8);
out[2] = (uint8_t)(v >> 16);
out[3] = (uint8_t)(v >> 24);
out[4] = (uint8_t)(v >> 32);
out[5] = (uint8_t)(v >> 40);
out[6] = (uint8_t)(v >> 48);
out[7] = (uint8_t)(v >> 56);
}
#endif /* BROTLI_LITTLE_ENDIAN */
#endif
#define TEMPLATE_(T) \

4
C/brotli/enc/write_bits.h
View File

@@ -40,7 +40,7 @@ static BROTLI_INLINE void BrotliWriteBits(size_t n_bits,
uint64_t bits,
size_t * BROTLI_RESTRICT pos,
uint8_t * BROTLI_RESTRICT array) {
#ifdef IS_LITTLE_ENDIAN
#ifdef BROTLI_LITTLE_ENDIAN
/* This branch of the code can write up to 56 bits at a time,
7 bits are lost by being perhaps already in *p and at least
1 bit is needed to initialize the bit-stream ahead (i.e. if 7
@@ -54,7 +54,7 @@ static BROTLI_INLINE void BrotliWriteBits(size_t n_bits,
assert((bits >> n_bits) == 0);
assert(n_bits <= 56);
v |= bits << (*pos & 7);
BROTLI_UNALIGNED_STORE64(p, v); /* Set some bits. */
BROTLI_UNALIGNED_STORE64LE(p, v); /* Set some bits. */
*pos += n_bits;
#else
/* implicit & 0xff is assumed for uint8_t arithmetics */

33
C/brotli/encode.h
View File

@@ -72,7 +72,9 @@ typedef enum BrotliEncoderOperation {
* Actual flush is performed when input stream is depleted and there is enough
* space in output stream. This means that client should repeat
* ::BROTLI_OPERATION_FLUSH operation until @p available_in becomes @c 0, and
* ::BrotliEncoderHasMoreOutput returns ::BROTLI_FALSE.
* ::BrotliEncoderHasMoreOutput returns ::BROTLI_FALSE. If output is acquired
* via ::BrotliEncoderTakeOutput, then operation should be repeated after
* output buffer is drained.
*
* @warning Until flush is complete, client @b SHOULD @b NOT swap,
* reduce or extend input stream.
@@ -86,8 +88,10 @@ typedef enum BrotliEncoderOperation {
*
* Actual finalization is performed when input stream is depleted and there is
* enough space in output stream. This means that client should repeat
* ::BROTLI_OPERATION_FLUSH operation until @p available_in becomes @c 0, and
* ::BrotliEncoderHasMoreOutput returns ::BROTLI_FALSE.
* ::BROTLI_OPERATION_FINISH operation until @p available_in becomes @c 0, and
* ::BrotliEncoderHasMoreOutput returns ::BROTLI_FALSE. If output is acquired
* via ::BrotliEncoderTakeOutput, then operation should be repeated after
* output buffer is drained.
*
* @warning Until finalization is complete, client @b SHOULD @b NOT swap,
* reduce or extend input stream.
@@ -223,29 +227,6 @@ BROTLI_ENC_API BrotliEncoderState* BrotliEncoderCreateInstance(
*/
BROTLI_ENC_API void BrotliEncoderDestroyInstance(BrotliEncoderState* state);
/**
* Prepends imaginary LZ77 dictionary.
*
* Fills the fresh ::BrotliEncoderState with additional data corpus for LZ77
* backward references.
*
* @note Not to be confused with the static dictionary (see RFC7932 section 8).
*
* Workflow:
* -# Allocate and initialize state with ::BrotliEncoderCreateInstance
* -# Set ::BROTLI_PARAM_LGWIN parameter
* -# Invoke ::BrotliEncoderSetCustomDictionary
* -# Use ::BrotliEncoderCompressStream
* -# Clean up and free state with ::BrotliEncoderDestroyInstance
*
* @param state encoder instance
* @param size length of @p dict; at most "window size" bytes are used
* @param dict "dictionary"; @b MUST use same dictionary during decompression
*/
BROTLI_ENC_API void BrotliEncoderSetCustomDictionary(
BrotliEncoderState* state, size_t size,
const uint8_t dict[BROTLI_ARRAY_PARAM(size)]);
/**
* Calculates the output size bound for the given @p input_size.
*

11
C/brotli/port.h
View File

@@ -117,11 +117,14 @@ OR:
#define BROTLI_INLINE
#endif
#else /* _MSC_VER */
# pragma warning(disable : 4100)
# pragma warning(disable : 4127)
# pragma warning(disable : 4389)
# pragma warning(disable : 4701)
#define BROTLI_INLINE __forceinline
/* disable some warnings */
#pragma warning(disable : 4100)
#pragma warning(disable : 4127)
#pragma warning(disable : 4189)
#pragma warning(disable : 4389)
#pragma warning(disable : 4701)
#endif /* _MSC_VER */
#if !defined(__cplusplus) && !defined(c_plusplus) && \

4
C/zstdmt/brotli-mt.h
View File

@@ -32,8 +32,8 @@ extern "C" {
#define BROTLIMT_MAGICNUMBER 0x5242U /* BR */
#define BROTLIMT_MAGIC_SKIPPABLE 0x184D2A50U
#define BROTLI_VERSION_MAJOR 0
#define BROTLI_VERSION_MINOR 6
#define BROTLI_VERSION_MAJOR 1
#define BROTLI_VERSION_MINOR 0
/* **************************************
* Error Handling