xevion/easy7zip
1
0
Fork 0
mirror of https://github.com/Xevion/easy7zip.git synced 2025-12-15 04:11:46 -06:00
Code Issues Packages Projects Releases Wiki Activity

update Brotli to version 1.0.6

Browse Source
This commit is contained in:
Tino Reichardt
2018-10-21 13:33:23 +02:00
parent e8cb55330d
commit 51dc99984a
82 changed files with 3537 additions and 2397 deletions
Download Patch File Download Diff File Expand all files Collapse all files

11
C/brotli/enc/backward_references.h
View File

@@ -11,10 +11,10 @@
#include "../common/constants.h"
#include "../common/dictionary.h"
#include "../common/platform.h"
#include "../types.h"
#include "./command.h"
#include "./hash.h"
#include "./port.h"
#include "./quality.h"
#if defined(__cplusplus) || defined(c_plusplus)
@@ -26,11 +26,10 @@ extern "C" {
CreateBackwardReferences calls, and must be incremented by the amount written
by this call. */
BROTLI_INTERNAL void BrotliCreateBackwardReferences(
const BrotliDictionary* dictionary, size_t num_bytes, size_t position,
const uint8_t* ringbuffer, size_t ringbuffer_mask,
const BrotliEncoderParams* params, HasherHandle hasher, int* dist_cache,
size_t* last_insert_len, Command* commands, size_t* num_commands,
size_t* num_literals);
size_t num_bytes, size_t position, const uint8_t* ringbuffer,
size_t ringbuffer_mask, const BrotliEncoderParams* params,
HasherHandle hasher, int* dist_cache, size_t* last_insert_len,
Command* commands, size_t* num_commands, size_t* num_literals);
#if defined(__cplusplus) || defined(c_plusplus)
} /* extern "C" */

47
C/brotli/enc/backward_references_hq.h
View File

@@ -11,41 +11,38 @@
#include "../common/constants.h"
#include "../common/dictionary.h"
#include "../common/platform.h"
#include "../types.h"
#include "./command.h"
#include "./hash.h"
#include "./memory.h"
#include "./port.h"
#include "./quality.h"
#if defined(__cplusplus) || defined(c_plusplus)
extern "C" {
#endif
BROTLI_INTERNAL void BrotliCreateZopfliBackwardReferences(
MemoryManager* m, const BrotliDictionary* dictionary, size_t num_bytes,
size_t position, const uint8_t* ringbuffer, size_t ringbuffer_mask,
const BrotliEncoderParams* params, HasherHandle hasher, int* dist_cache,
size_t* last_insert_len, Command* commands, size_t* num_commands,
size_t* num_literals);
BROTLI_INTERNAL void BrotliCreateZopfliBackwardReferences(MemoryManager* m,
size_t num_bytes, size_t position, const uint8_t* ringbuffer,
size_t ringbuffer_mask, const BrotliEncoderParams* params,
HasherHandle hasher, int* dist_cache, size_t* last_insert_len,
Command* commands, size_t* num_commands, size_t* num_literals);
BROTLI_INTERNAL void BrotliCreateHqZopfliBackwardReferences(
MemoryManager* m, const BrotliDictionary* dictionary, size_t num_bytes,
size_t position, const uint8_t* ringbuffer, size_t ringbuffer_mask,
const BrotliEncoderParams* params, HasherHandle hasher, int* dist_cache,
size_t* last_insert_len, Command* commands, size_t* num_commands,
size_t* num_literals);
BROTLI_INTERNAL void BrotliCreateHqZopfliBackwardReferences(MemoryManager* m,
size_t num_bytes, size_t position, const uint8_t* ringbuffer,
size_t ringbuffer_mask, const BrotliEncoderParams* params,
HasherHandle hasher, int* dist_cache, size_t* last_insert_len,
Command* commands, size_t* num_commands, size_t* num_literals);
typedef struct ZopfliNode {
/* best length to get up to this byte (not including this byte itself)
highest 8 bit is used to reconstruct the length code */
/* Best length to get up to this byte (not including this byte itself)
highest 7 bit is used to reconstruct the length code. */
uint32_t length;
/* distance associated with the length
highest 7 bit contains distance short code + 1 (or zero if no short code)
*/
/* Distance associated with the length. */
uint32_t distance;
/* number of literal inserts before this copy */
uint32_t insert_length;
/* Number of literal inserts before this copy; highest 5 bits contain
distance short code + 1 (or zero if no short code). */
uint32_t dcode_insert_length;
/* This union holds information used by dynamic-programming. During forward
pass |cost| it used to store the goal function. When node is processed its
@@ -77,11 +74,11 @@ BROTLI_INTERNAL void BrotliInitZopfliNodes(ZopfliNode* array, size_t length);
(1) nodes[i].copy_length() >= 2
(2) nodes[i].command_length() <= i and
(3) nodes[i - nodes[i].command_length()].cost < kInfinity */
BROTLI_INTERNAL size_t BrotliZopfliComputeShortestPath(
MemoryManager* m, const BrotliDictionary* dictionary, size_t num_bytes,
size_t position, const uint8_t* ringbuffer, size_t ringbuffer_mask,
const BrotliEncoderParams* params, const size_t max_backward_limit,
const int* dist_cache, HasherHandle hasher, ZopfliNode* nodes);
BROTLI_INTERNAL size_t BrotliZopfliComputeShortestPath(MemoryManager* m,
size_t num_bytes, size_t position, const uint8_t* ringbuffer,
size_t ringbuffer_mask, const BrotliEncoderParams* params,
const size_t max_backward_limit, const int* dist_cache, HasherHandle hasher,
ZopfliNode* nodes);
BROTLI_INTERNAL void BrotliZopfliCreateCommands(
const size_t num_bytes, const size_t block_start,

20
C/brotli/enc/backward_references_inc.h
View File

@@ -8,8 +8,7 @@
/* template parameters: EXPORT_FN, FN */
static BROTLI_NOINLINE void EXPORT_FN(CreateBackwardReferences)(
const BrotliDictionary* dictionary,
const uint16_t* dictionary_hash, size_t num_bytes, size_t position,
size_t num_bytes, size_t position,
const uint8_t* ringbuffer, size_t ringbuffer_mask,
const BrotliEncoderParams* params, HasherHandle hasher, int* dist_cache,
size_t* last_insert_len, Command* commands, size_t* num_commands,
@@ -42,9 +41,10 @@ static BROTLI_NOINLINE void EXPORT_FN(CreateBackwardReferences)(
sr.len_code_delta = 0;
sr.distance = 0;
sr.score = kMinScore;
FN(FindLongestMatch)(hasher, dictionary, dictionary_hash, ringbuffer,
ringbuffer_mask, dist_cache, position,
max_length, max_distance, gap, &sr);
FN(FindLongestMatch)(hasher, &params->dictionary,
ringbuffer, ringbuffer_mask, dist_cache, position,
max_length, max_distance, gap,
params->dist.max_distance, &sr);
if (sr.score > kMinScore) {
/* Found a match. Let's look for something even better ahead. */
int delayed_backward_references_in_row = 0;
@@ -58,9 +58,9 @@ static BROTLI_NOINLINE void EXPORT_FN(CreateBackwardReferences)(
sr2.distance = 0;
sr2.score = kMinScore;
max_distance = BROTLI_MIN(size_t, position + 1, max_backward_limit);
FN(FindLongestMatch)(hasher, dictionary, dictionary_hash, ringbuffer,
ringbuffer_mask, dist_cache, position + 1,
max_length, max_distance, gap, &sr2);
FN(FindLongestMatch)(hasher, &params->dictionary,
ringbuffer, ringbuffer_mask, dist_cache, position + 1, max_length,
max_distance, gap, params->dist.max_distance, &sr2);
if (sr2.score >= sr.score + cost_diff_lazy) {
/* Ok, let's just write one byte for now and start a match from the
next byte. */
@@ -89,8 +89,8 @@ static BROTLI_NOINLINE void EXPORT_FN(CreateBackwardReferences)(
dist_cache[0] = (int)sr.distance;
FN(PrepareDistanceCache)(hasher, dist_cache);
}
InitCommand(commands++, insert_length, sr.len, sr.len_code_delta,
distance_code);
InitCommand(commands++, &params->dist, insert_length,
sr.len, sr.len_code_delta, distance_code);
}
*num_literals += insert_length;
insert_length = 0;

10
C/brotli/enc/bit_cost.h
View File

@@ -9,20 +9,20 @@
#ifndef BROTLI_ENC_BIT_COST_H_
#define BROTLI_ENC_BIT_COST_H_
#include "../common/platform.h"
#include "../types.h"
#include "./fast_log.h"
#include "./histogram.h"
#include "./port.h"
#if defined(__cplusplus) || defined(c_plusplus)
extern "C" {
#endif
static BROTLI_INLINE double ShannonEntropy(const uint32_t *population,
size_t size, size_t *total) {
static BROTLI_INLINE double ShannonEntropy(
const uint32_t* population, size_t size, size_t* total) {
size_t sum = 0;
double retval = 0;
const uint32_t *population_end = population + size;
const uint32_t* population_end = population + size;
size_t p;
if (size & 1) {
goto odd_number_of_elements_left;
@@ -42,7 +42,7 @@ static BROTLI_INLINE double ShannonEntropy(const uint32_t *population,
}
static BROTLI_INLINE double BitsEntropy(
const uint32_t *population, size_t size) {
const uint32_t* population, size_t size) {
size_t sum;
double retval = ShannonEntropy(population, size, &sum);
if (retval < sum) {

13
C/brotli/enc/block_encoder_inc.h
View File

@@ -13,9 +13,9 @@
stream. */
static void FN(BuildAndStoreEntropyCodes)(MemoryManager* m, BlockEncoder* self,
const HistogramType* histograms, const size_t histograms_size,
HuffmanTree* tree, size_t* storage_ix, uint8_t* storage) {
const size_t alphabet_size = self->alphabet_size_;
const size_t table_size = histograms_size * alphabet_size;
const size_t alphabet_size, HuffmanTree* tree,
size_t* storage_ix, uint8_t* storage) {
const size_t table_size = histograms_size * self->histogram_length_;
self->depths_ = BROTLI_ALLOC(m, uint8_t, table_size);
self->bits_ = BROTLI_ALLOC(m, uint16_t, table_size);
if (BROTLI_IS_OOM(m)) return;
@@ -23,9 +23,10 @@ static void FN(BuildAndStoreEntropyCodes)(MemoryManager* m, BlockEncoder* self,
{
size_t i;
for (i = 0; i < histograms_size; ++i) {
size_t ix = i * alphabet_size;
BuildAndStoreHuffmanTree(&histograms[i].data_[0], alphabet_size, tree,
&self->depths_[ix], &self->bits_[ix], storage_ix, storage);
size_t ix = i * self->histogram_length_;
BuildAndStoreHuffmanTree(&histograms[i].data_[0], self->histogram_length_,
alphabet_size, tree, &self->depths_[ix], &self->bits_[ix],
storage_ix, storage);
}
}
}

2
C/brotli/enc/block_splitter.h
View File

@@ -9,10 +9,10 @@
#ifndef BROTLI_ENC_BLOCK_SPLITTER_H_
#define BROTLI_ENC_BLOCK_SPLITTER_H_
#include "../common/platform.h"
#include "../types.h"
#include "./command.h"
#include "./memory.h"
#include "./port.h"
#include "./quality.h"
#if defined(__cplusplus) || defined(c_plusplus)

22
C/brotli/enc/block_splitter_inc.h
View File

@@ -70,13 +70,13 @@ static size_t FN(FindBlocks)(const DataType* data, const size_t length,
double* insert_cost,
double* cost,
uint8_t* switch_signal,
uint8_t *block_id) {
uint8_t* block_id) {
const size_t data_size = FN(HistogramDataSize)();
const size_t bitmaplen = (num_histograms + 7) >> 3;
size_t num_blocks = 1;
size_t i;
size_t j;
assert(num_histograms <= 256);
BROTLI_DCHECK(num_histograms <= 256);
if (num_histograms <= 1) {
for (i = 0; i < length; ++i) {
block_id[i] = 0;
@@ -126,7 +126,7 @@ static size_t FN(FindBlocks)(const DataType* data, const size_t length,
if (cost[k] >= block_switch_cost) {
const uint8_t mask = (uint8_t)(1u << (k & 7));
cost[k] = block_switch_cost;
assert((k >> 3) < bitmaplen);
BROTLI_DCHECK((k >> 3) < bitmaplen);
switch_signal[ix + (k >> 3)] |= mask;
}
}
@@ -137,7 +137,7 @@ static size_t FN(FindBlocks)(const DataType* data, const size_t length,
uint8_t cur_id = block_id[byte_ix];
while (byte_ix > 0) {
const uint8_t mask = (uint8_t)(1u << (cur_id & 7));
assert(((size_t)cur_id >> 3) < bitmaplen);
BROTLI_DCHECK(((size_t)cur_id >> 3) < bitmaplen);
--byte_ix;
ix -= bitmaplen;
if (switch_signal[ix + (cur_id >> 3)] & mask) {
@@ -161,16 +161,16 @@ static size_t FN(RemapBlockIds)(uint8_t* block_ids, const size_t length,
new_id[i] = kInvalidId;
}
for (i = 0; i < length; ++i) {
assert(block_ids[i] < num_histograms);
BROTLI_DCHECK(block_ids[i] < num_histograms);
if (new_id[block_ids[i]] == kInvalidId) {
new_id[block_ids[i]] = next_id++;
}
}
for (i = 0; i < length; ++i) {
block_ids[i] = (uint8_t)new_id[block_ids[i]];
assert(block_ids[i] < num_histograms);
BROTLI_DCHECK(block_ids[i] < num_histograms);
}
assert(next_id <= num_histograms);
BROTLI_DCHECK(next_id <= num_histograms);
return next_id;
}
@@ -226,13 +226,13 @@ static void FN(ClusterBlocks)(MemoryManager* m,
{
size_t block_idx = 0;
for (i = 0; i < length; ++i) {
assert(block_idx < num_blocks);
BROTLI_DCHECK(block_idx < num_blocks);
++block_lengths[block_idx];
if (i + 1 == length || block_ids[i] != block_ids[i + 1]) {
++block_idx;
}
}
assert(block_idx == num_blocks);
BROTLI_DCHECK(block_idx == num_blocks);
}
for (i = 0; i < num_blocks; i += HISTOGRAMS_PER_BATCH) {
@@ -268,8 +268,8 @@ static void FN(ClusterBlocks)(MemoryManager* m,
histogram_symbols[i + j] = (uint32_t)num_clusters + remap[symbols[j]];
}
num_clusters += num_new_clusters;
assert(num_clusters == cluster_size_size);
assert(num_clusters == all_histograms_size);
BROTLI_DCHECK(num_clusters == cluster_size_size);
BROTLI_DCHECK(num_clusters == all_histograms_size);
}
BROTLI_FREE(m, histograms);

57
C/brotli/enc/brotli_bit_stream.h
View File

@@ -16,13 +16,13 @@
#ifndef BROTLI_ENC_BROTLI_BIT_STREAM_H_
#define BROTLI_ENC_BROTLI_BIT_STREAM_H_
#include "../common/context.h"
#include "../common/platform.h"
#include "../types.h"
#include "./command.h"
#include "./context.h"
#include "./entropy_encode.h"
#include "./memory.h"
#include "./metablock.h"
#include "./port.h"
#if defined(__cplusplus) || defined(c_plusplus)
extern "C" {
@@ -32,7 +32,7 @@ extern "C" {
position for the current storage. */
BROTLI_INTERNAL void BrotliStoreHuffmanTree(const uint8_t* depths, size_t num,
HuffmanTree* tree, size_t *storage_ix, uint8_t *storage);
HuffmanTree* tree, size_t* storage_ix, uint8_t* storage);
BROTLI_INTERNAL void BrotliBuildAndStoreHuffmanTreeFast(
MemoryManager* m, const uint32_t* histogram, const size_t histogram_total,
@@ -42,59 +42,40 @@ BROTLI_INTERNAL void BrotliBuildAndStoreHuffmanTreeFast(
/* REQUIRES: length > 0 */
/* REQUIRES: length <= (1 << 24) */
BROTLI_INTERNAL void BrotliStoreMetaBlock(MemoryManager* m,
const uint8_t* input,
size_t start_pos,
size_t length,
size_t mask,
uint8_t prev_byte,
uint8_t prev_byte2,
BROTLI_BOOL is_final_block,
uint32_t num_direct_distance_codes,
uint32_t distance_postfix_bits,
ContextType literal_context_mode,
const Command* commands,
size_t n_commands,
const MetaBlockSplit* mb,
size_t* storage_ix,
uint8_t* storage);
const uint8_t* input, size_t start_pos, size_t length, size_t mask,
uint8_t prev_byte, uint8_t prev_byte2, BROTLI_BOOL is_last,
const BrotliEncoderParams* params, ContextType literal_context_mode,
const Command* commands, size_t n_commands, const MetaBlockSplit* mb,
size_t* storage_ix, uint8_t* storage);
/* Stores the meta-block without doing any block splitting, just collects
one histogram per block category and uses that for entropy coding.
REQUIRES: length > 0
REQUIRES: length <= (1 << 24) */
BROTLI_INTERNAL void BrotliStoreMetaBlockTrivial(MemoryManager* m,
const uint8_t* input,
size_t start_pos,
size_t length,
size_t mask,
BROTLI_BOOL is_last,
const Command *commands,
size_t n_commands,
size_t* storage_ix,
uint8_t* storage);
const uint8_t* input, size_t start_pos, size_t length, size_t mask,
BROTLI_BOOL is_last, const BrotliEncoderParams* params,
const Command* commands, size_t n_commands,
size_t* storage_ix, uint8_t* storage);
/* Same as above, but uses static prefix codes for histograms with a only a few
symbols, and uses static code length prefix codes for all other histograms.
REQUIRES: length > 0
REQUIRES: length <= (1 << 24) */
BROTLI_INTERNAL void BrotliStoreMetaBlockFast(MemoryManager* m,
const uint8_t* input,
size_t start_pos,
size_t length,
size_t mask,
BROTLI_BOOL is_last,
const Command *commands,
size_t n_commands,
size_t* storage_ix,
uint8_t* storage);
const uint8_t* input, size_t start_pos, size_t length, size_t mask,
BROTLI_BOOL is_last, const BrotliEncoderParams* params,
const Command* commands, size_t n_commands,
size_t* storage_ix, uint8_t* storage);
/* This is for storing uncompressed blocks (simple raw storage of
bytes-as-bytes).
REQUIRES: length > 0
REQUIRES: length <= (1 << 24) */
BROTLI_INTERNAL void BrotliStoreUncompressedMetaBlock(
BROTLI_BOOL is_final_block, const uint8_t* input, size_t position,
size_t mask, size_t len, size_t* storage_ix, uint8_t* storage);
BROTLI_BOOL is_final_block, const uint8_t* BROTLI_RESTRICT input,
size_t position, size_t mask, size_t len,
size_t* BROTLI_RESTRICT storage_ix, uint8_t* BROTLI_RESTRICT storage);
#if defined(__cplusplus) || defined(c_plusplus)
} /* extern "C" */

2
C/brotli/enc/cluster.h
View File

@@ -9,10 +9,10 @@
#ifndef BROTLI_ENC_CLUSTER_H_
#define BROTLI_ENC_CLUSTER_H_
#include "../common/platform.h"
#include "../types.h"
#include "./histogram.h"
#include "./memory.h"
#include "./port.h"
#if defined(__cplusplus) || defined(c_plusplus)
extern "C" {

70
C/brotli/enc/command.h
View File

@@ -10,9 +10,10 @@
#define BROTLI_ENC_COMMAND_H_
#include "../common/constants.h"
#include "../port.h"
#include "../common/platform.h"
#include "../types.h"
#include "./fast_log.h"
#include "./params.h"
#include "./prefix.h"
#if defined(__cplusplus) || defined(c_plusplus)
@@ -61,21 +62,21 @@ static BROTLI_INLINE uint16_t GetCopyLengthCode(size_t copylen) {
static BROTLI_INLINE uint16_t CombineLengthCodes(
uint16_t inscode, uint16_t copycode, BROTLI_BOOL use_last_distance) {
uint16_t bits64 =
(uint16_t)((copycode & 0x7u) | ((inscode & 0x7u) << 3));
if (use_last_distance && inscode < 8 && copycode < 16) {
return (copycode < 8) ? bits64 : (bits64 | 64);
(uint16_t)((copycode & 0x7u) | ((inscode & 0x7u) << 3u));
if (use_last_distance && inscode < 8u && copycode < 16u) {
return (copycode < 8u) ? bits64 : (bits64 | 64u);
} else {
/* Specification: 5 Encoding of ... (last table) */
/* offset = 2 * index, where index is in range [0..8] */
int offset = 2 * ((copycode >> 3) + 3 * (inscode >> 3));
uint32_t offset = 2u * ((copycode >> 3u) + 3u * (inscode >> 3u));
/* All values in specification are K * 64,
where K = [2, 3, 6, 4, 5, 8, 7, 9, 10],
i + 1 = [1, 2, 3, 4, 5, 6, 7, 8, 9],
K - i - 1 = [1, 1, 3, 0, 0, 2, 0, 1, 2] = D.
All values in D require only 2 bits to encode.
Magic constant is shifted 6 bits left, to avoid final multiplication. */
offset = (offset << 5) + 0x40 + ((0x520D40 >> offset) & 0xC0);
return (uint16_t)offset | bits64;
offset = (offset << 5u) + 0x40u + ((0x520D40u >> offset) & 0xC0u);
return (uint16_t)(offset | bits64);
}
}
@@ -105,53 +106,61 @@ static BROTLI_INLINE uint32_t GetCopyExtra(uint16_t copycode) {
typedef struct Command {
uint32_t insert_len_;
/* Stores copy_len in low 24 bits and copy_len XOR copy_code in high 8 bit. */
/* Stores copy_len in low 25 bits and copy_code - copy_len in high 7 bit. */
uint32_t copy_len_;
/* Stores distance extra bits. */
uint32_t dist_extra_;
uint16_t cmd_prefix_;
/* Stores distance code in low 10 bits
and number of extra bits in high 6 bits. */
uint16_t dist_prefix_;
} Command;
/* distance_code is e.g. 0 for same-as-last short code, or 16 for offset 1. */
static BROTLI_INLINE void InitCommand(Command* self, size_t insertlen,
static BROTLI_INLINE void InitCommand(Command* self,
const BrotliDistanceParams* dist, size_t insertlen,
size_t copylen, int copylen_code_delta, size_t distance_code) {
/* Don't rely on signed int representation, use honest casts. */
uint32_t delta = (uint8_t)((int8_t)copylen_code_delta);
self->insert_len_ = (uint32_t)insertlen;
self->copy_len_ = (uint32_t)(copylen | (delta << 24));
self->copy_len_ = (uint32_t)(copylen | (delta << 25));
/* The distance prefix and extra bits are stored in this Command as if
npostfix and ndirect were 0, they are only recomputed later after the
clustering if needed. */
PrefixEncodeCopyDistance(
distance_code, 0, 0, &self->dist_prefix_, &self->dist_extra_);
distance_code, dist->num_direct_distance_codes,
dist->distance_postfix_bits, &self->dist_prefix_, &self->dist_extra_);
GetLengthCode(
insertlen, (size_t)((int)copylen + copylen_code_delta),
TO_BROTLI_BOOL(self->dist_prefix_ == 0), &self->cmd_prefix_);
TO_BROTLI_BOOL((self->dist_prefix_ & 0x3FF) == 0), &self->cmd_prefix_);
}
static BROTLI_INLINE void InitInsertCommand(Command* self, size_t insertlen) {
self->insert_len_ = (uint32_t)insertlen;
self->copy_len_ = 4 << 24;
self->copy_len_ = 4 << 25;
self->dist_extra_ = 0;
self->dist_prefix_ = BROTLI_NUM_DISTANCE_SHORT_CODES;
GetLengthCode(insertlen, 4, BROTLI_FALSE, &self->cmd_prefix_);
}
static BROTLI_INLINE uint32_t CommandRestoreDistanceCode(const Command* self) {
if (self->dist_prefix_ < BROTLI_NUM_DISTANCE_SHORT_CODES) {
return self->dist_prefix_;
static BROTLI_INLINE uint32_t CommandRestoreDistanceCode(
const Command* self, const BrotliDistanceParams* dist) {
if ((self->dist_prefix_ & 0x3FFu) <
BROTLI_NUM_DISTANCE_SHORT_CODES + dist->num_direct_distance_codes) {
return self->dist_prefix_ & 0x3FFu;
} else {
uint32_t nbits = self->dist_extra_ >> 24;
uint32_t extra = self->dist_extra_ & 0xffffff;
/* It is assumed that the distance was first encoded with NPOSTFIX = 0 and
NDIRECT = 0, so the code itself is of this form:
BROTLI_NUM_DISTANCE_SHORT_CODES + 2 * (nbits - 1) + prefix_bit
Therefore, the following expression results in (2 + prefix_bit). */
uint32_t prefix =
self->dist_prefix_ + 4u - BROTLI_NUM_DISTANCE_SHORT_CODES - 2u * nbits;
/* Subtract 4 for offset (Chapter 4.) and
increase by BROTLI_NUM_DISTANCE_SHORT_CODES - 1 */
return (prefix << nbits) + extra + BROTLI_NUM_DISTANCE_SHORT_CODES - 4u;
uint32_t dcode = self->dist_prefix_ & 0x3FFu;
uint32_t nbits = self->dist_prefix_ >> 10;
uint32_t extra = self->dist_extra_;
uint32_t postfix_mask = (1U << dist->distance_postfix_bits) - 1U;
uint32_t hcode = (dcode - dist->num_direct_distance_codes -
BROTLI_NUM_DISTANCE_SHORT_CODES) >>
dist->distance_postfix_bits;
uint32_t lcode = (dcode - dist->num_direct_distance_codes -
BROTLI_NUM_DISTANCE_SHORT_CODES) & postfix_mask;
uint32_t offset = ((2U + (hcode & 1U)) << nbits) - 4U;
return ((offset + extra) << dist->distance_postfix_bits) + lcode +
dist->num_direct_distance_codes + BROTLI_NUM_DISTANCE_SHORT_CODES;
}
}
@@ -165,12 +174,13 @@ static BROTLI_INLINE uint32_t CommandDistanceContext(const Command* self) {
}
static BROTLI_INLINE uint32_t CommandCopyLen(const Command* self) {
return self->copy_len_ & 0xFFFFFF;
return self->copy_len_ & 0x1FFFFFF;
}
static BROTLI_INLINE uint32_t CommandCopyLenCode(const Command* self) {
int32_t delta = (int8_t)((uint8_t)(self->copy_len_ >> 24));
return (uint32_t)((int32_t)(self->copy_len_ & 0xFFFFFF) + delta);
uint32_t modifier = self->copy_len_ >> 25;
int32_t delta = (int8_t)((uint8_t)(modifier | ((modifier & 0x40) << 1)));
return (uint32_t)((int32_t)(self->copy_len_ & 0x1FFFFFF) + delta);
}
#if defined(__cplusplus) || defined(c_plusplus)

2
C/brotli/enc/compress_fragment.h
View File

@@ -12,9 +12,9 @@
#ifndef BROTLI_ENC_COMPRESS_FRAGMENT_H_
#define BROTLI_ENC_COMPRESS_FRAGMENT_H_
#include "../common/platform.h"
#include "../types.h"
#include "./memory.h"
#include "./port.h"
#if defined(__cplusplus) || defined(c_plusplus)
extern "C" {

2
C/brotli/enc/compress_fragment_two_pass.h
View File

@@ -13,9 +13,9 @@
#ifndef BROTLI_ENC_COMPRESS_FRAGMENT_TWO_PASS_H_
#define BROTLI_ENC_COMPRESS_FRAGMENT_TWO_PASS_H_
#include "../common/platform.h"
#include "../types.h"
#include "./memory.h"
#include "./port.h"
#if defined(__cplusplus) || defined(c_plusplus)
extern "C" {

184
C/brotli/enc/context.h
View File

@@ -1,184 +0,0 @@
/* Copyright 2013 Google Inc. All Rights Reserved.
Distributed under MIT license.
See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
*/
/* Functions to map previous bytes into a context id. */
#ifndef BROTLI_ENC_CONTEXT_H_
#define BROTLI_ENC_CONTEXT_H_
#include "../port.h"
#include "../types.h"
#if defined(__cplusplus) || defined(c_plusplus)
extern "C" {
#endif
/* Second-order context lookup table for UTF8 byte streams.
If p1 and p2 are the previous two bytes, we calculate the context as
context = kUTF8ContextLookup[p1] | kUTF8ContextLookup[p2 + 256].
If the previous two bytes are ASCII characters (i.e. < 128), this will be
equivalent to
context = 4 * context1(p1) + context2(p2),
where context1 is based on the previous byte in the following way:
0 : non-ASCII control
1 : \t, \n, \r
2 : space
3 : other punctuation
4 : " '
5 : %
6 : ( < [ {
7 : ) > ] }
8 : , ; :
9 : .
10 : =
11 : number
12 : upper-case vowel
13 : upper-case consonant
14 : lower-case vowel
15 : lower-case consonant
and context2 is based on the second last byte:
0 : control, space
1 : punctuation
2 : upper-case letter, number
3 : lower-case letter
If the last byte is ASCII, and the second last byte is not (in a valid UTF8
stream it will be a continuation byte, value between 128 and 191), the
context is the same as if the second last byte was an ASCII control or space.
If the last byte is a UTF8 lead byte (value >= 192), then the next byte will
be a continuation byte and the context id is 2 or 3 depending on the LSB of
the last byte and to a lesser extent on the second last byte if it is ASCII.
If the last byte is a UTF8 continuation byte, the second last byte can be:
- continuation byte: the next byte is probably ASCII or lead byte (assuming
4-byte UTF8 characters are rare) and the context id is 0 or 1.
- lead byte (192 - 207): next byte is ASCII or lead byte, context is 0 or 1
- lead byte (208 - 255): next byte is continuation byte, context is 2 or 3
The possible value combinations of the previous two bytes, the range of
context ids and the type of the next byte is summarized in the table below:
|--------\-----------------------------------------------------------------|
| \ Last byte |
| Second \---------------------------------------------------------------|
| last byte \ ASCII | cont. byte | lead byte |
| \ (0-127) | (128-191) | (192-) |
|=============|===================|=====================|==================|
| ASCII | next: ASCII/lead | not valid | next: cont. |
| (0-127) | context: 4 - 63 | | context: 2 - 3 |
|-------------|-------------------|---------------------|------------------|
| cont. byte | next: ASCII/lead | next: ASCII/lead | next: cont. |
| (128-191) | context: 4 - 63 | context: 0 - 1 | context: 2 - 3 |
|-------------|-------------------|---------------------|------------------|
| lead byte | not valid | next: ASCII/lead | not valid |
| (192-207) | | context: 0 - 1 | |
|-------------|-------------------|---------------------|------------------|
| lead byte | not valid | next: cont. | not valid |
| (208-) | | context: 2 - 3 | |
|-------------|-------------------|---------------------|------------------|
*/
static const uint8_t kUTF8ContextLookup[512] = {
/* Last byte. */
/* */
/* ASCII range. */
0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 4, 0, 0, 4, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
8, 12, 16, 12, 12, 20, 12, 16, 24, 28, 12, 12, 32, 12, 36, 12,
44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 32, 32, 24, 40, 28, 12,
12, 48, 52, 52, 52, 48, 52, 52, 52, 48, 52, 52, 52, 52, 52, 48,
52, 52, 52, 52, 52, 48, 52, 52, 52, 52, 52, 24, 12, 28, 12, 12,
12, 56, 60, 60, 60, 56, 60, 60, 60, 56, 60, 60, 60, 60, 60, 56,
60, 60, 60, 60, 60, 56, 60, 60, 60, 60, 60, 24, 12, 28, 12, 0,
/* UTF8 continuation byte range. */
0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1,
0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1,
0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1,
0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1,
/* UTF8 lead byte range. */
2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3,
2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3,
2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3,
2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3,
/* Second last byte. */
/* */
/* ASCII range. */
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 1, 1, 1, 1, 1, 1,
1, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 1, 1, 1, 1, 1,
1, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3,
3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 1, 1, 1, 1, 0,
/* UTF8 continuation byte range. */
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
/* UTF8 lead byte range. */
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
};
/* Context lookup table for small signed integers. */
static const uint8_t kSigned3BitContextLookup[] = {
0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3,
3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3,
3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3,
3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3,
4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4,
4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4,
4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4,
4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4,
5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5,
5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5,
5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5,
6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 7,
};
typedef enum ContextType {
CONTEXT_LSB6 = 0,
CONTEXT_MSB6 = 1,
CONTEXT_UTF8 = 2,
CONTEXT_SIGNED = 3
} ContextType;
static BROTLI_INLINE uint8_t Context(uint8_t p1, uint8_t p2, ContextType mode) {
switch (mode) {
case CONTEXT_LSB6:
return p1 & 0x3f;
case CONTEXT_MSB6:
return (uint8_t)(p1 >> 2);
case CONTEXT_UTF8:
return kUTF8ContextLookup[p1] | kUTF8ContextLookup[p2 + 256];
case CONTEXT_SIGNED:
return (uint8_t)((kSigned3BitContextLookup[p1] << 3) +
kSigned3BitContextLookup[p2]);
default:
return 0;
}
}
#if defined(__cplusplus) || defined(c_plusplus)
} /* extern "C" */
#endif
#endif /* BROTLI_ENC_CONTEXT_H_ */

41
C/brotli/enc/encoder_dict.h Normal file
View File

@@ -0,0 +1,41 @@
/* Copyright 2017 Google Inc. All Rights Reserved.
Distributed under MIT license.
See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
*/
#ifndef BROTLI_ENC_ENCODER_DICT_H_
#define BROTLI_ENC_ENCODER_DICT_H_
#include "../common/dictionary.h"
#include "../common/platform.h"
#include "../types.h"
#include "./static_dict_lut.h"
#if defined(__cplusplus) || defined(c_plusplus)
extern "C" {
#endif
/* Dictionary data (words and transforms) for 1 possible context */
typedef struct BrotliEncoderDictionary {
const BrotliDictionary* words;
/* cut off for fast encoder */
uint32_t cutoffTransformsCount;
uint64_t cutoffTransforms;
/* from dictionary_hash.h, for fast encoder */
const uint16_t* hash_table;
/* from static_dict_lut.h, for slow encoder */
const uint16_t* buckets;
const DictWord* dict_words;
} BrotliEncoderDictionary;
BROTLI_INTERNAL void BrotliInitEncoderDictionary(BrotliEncoderDictionary* dict);
#if defined(__cplusplus) || defined(c_plusplus)
} /* extern "C" */
#endif
#endif /* BROTLI_ENC_ENCODER_DICT_H_ */

10
C/brotli/enc/entropy_encode.h
View File

@@ -9,8 +9,8 @@
#ifndef BROTLI_ENC_ENTROPY_ENCODE_H_
#define BROTLI_ENC_ENTROPY_ENCODE_H_
#include "../common/platform.h"
#include "../types.h"
#include "./port.h"
#if defined(__cplusplus) || defined(c_plusplus)
extern "C" {
@@ -46,11 +46,11 @@ BROTLI_INTERNAL BROTLI_BOOL BrotliSetDepth(
be at least 2 * length + 1 long.
See http://en.wikipedia.org/wiki/Huffman_coding */
BROTLI_INTERNAL void BrotliCreateHuffmanTree(const uint32_t *data,
BROTLI_INTERNAL void BrotliCreateHuffmanTree(const uint32_t* data,
const size_t length,
const int tree_limit,
HuffmanTree* tree,
uint8_t *depth);
uint8_t* depth);
/* Change the population counts in a way that the consequent
Huffman tree compression, especially its RLE-part will be more
@@ -72,9 +72,9 @@ BROTLI_INTERNAL void BrotliWriteHuffmanTree(const uint8_t* depth,
uint8_t* extra_bits_data);
/* Get the actual bit values for a tree of bit depths. */
BROTLI_INTERNAL void BrotliConvertBitDepthsToSymbols(const uint8_t *depth,
BROTLI_INTERNAL void BrotliConvertBitDepthsToSymbols(const uint8_t* depth,
size_t len,
uint16_t *bits);
uint16_t* bits);
/* Input size optimized Shell sort. */
typedef BROTLI_BOOL (*HuffmanTreeComparator)(

6
C/brotli/enc/entropy_encode_static.h
View File

@@ -10,7 +10,7 @@
#define BROTLI_ENC_ENTROPY_ENCODE_STATIC_H_
#include "../common/constants.h"
#include "../port.h"
#include "../common/platform.h"
#include "../types.h"
#include "./write_bits.h"
@@ -83,7 +83,7 @@ static const uint32_t kCodeLengthBits[18] = {
static BROTLI_INLINE void StoreStaticCodeLengthCode(
size_t* storage_ix, uint8_t* storage) {
BrotliWriteBits(
40, BROTLI_MAKE_UINT64_T(0x0000ffU, 0x55555554U), storage_ix, storage);
40, BROTLI_MAKE_UINT64_T(0x0000FFu, 0x55555554u), storage_ix, storage);
}
static const uint64_t kZeroRepsBits[BROTLI_NUM_COMMAND_SYMBOLS] = {
@@ -529,7 +529,7 @@ static const uint16_t kStaticDistanceCodeBits[64] = {
static BROTLI_INLINE void StoreStaticDistanceHuffmanTree(
size_t* storage_ix, uint8_t* storage) {
BrotliWriteBits(28, 0x0369dc03U, storage_ix, storage);
BrotliWriteBits(28, 0x0369DC03u, storage_ix, storage);
}
#if defined(__cplusplus) || defined(c_plusplus)

6
C/brotli/enc/fast_log.h
View File

@@ -11,15 +11,17 @@
#include <math.h>
#include "../common/platform.h"
#include "../types.h"
#include "../port.h"
#if defined(__cplusplus) || defined(c_plusplus)
extern "C" {
#endif
static BROTLI_INLINE uint32_t Log2FloorNonZero(size_t n) {
#if BROTLI_MODERN_COMPILER || __has_builtin(__builtin_clz)
/* TODO: generalize and move to platform.h */
#if BROTLI_GNUC_HAS_BUILTIN(__builtin_clz, 3, 4, 0) || \
BROTLI_INTEL_VERSION_CHECK(16, 0, 0)
return 31u ^ (uint32_t)__builtin_clz((uint32_t)n);
#else
uint32_t result = 0;

6
C/brotli/enc/find_match_length.h
View File

@@ -9,8 +9,8 @@
#ifndef BROTLI_ENC_FIND_MATCH_LENGTH_H_
#define BROTLI_ENC_FIND_MATCH_LENGTH_H_
#include "../common/platform.h"
#include "../types.h"
#include "./port.h"
#if defined(__cplusplus) || defined(c_plusplus)
extern "C" {
@@ -60,8 +60,8 @@ static BROTLI_INLINE size_t FindMatchLengthWithLimit(const uint8_t* s1,
the first non-matching bit and use that to calculate the total
length of the match. */
while (s2_ptr <= s2_limit - 4 &&
BROTLI_UNALIGNED_LOAD32(s2_ptr) ==
BROTLI_UNALIGNED_LOAD32(s1 + matched)) {
BrotliUnalignedRead32(s2_ptr) ==
BrotliUnalignedRead32(s1 + matched)) {
s2_ptr += 4;
matched += 4;
}

99
C/brotli/enc/hash.h
View File

@@ -14,11 +14,12 @@
#include "../common/constants.h"
#include "../common/dictionary.h"
#include "../common/platform.h"
#include "../types.h"
#include "./encoder_dict.h"
#include "./fast_log.h"
#include "./find_match_length.h"
#include "./memory.h"
#include "./port.h"
#include "./quality.h"
#include "./static_dict.h"
@@ -35,8 +36,10 @@ extern "C" {
* * HasherCommon structure
* * private structured hasher data, depending on hasher type
* * private dynamic hasher data, depending on hasher type and parameters
*/
typedef uint8_t* HasherHandle;
*
* Using "define" instead of "typedef", because on MSVC __restrict does not work
* on typedef pointer types. */
#define HasherHandle uint8_t*
typedef struct {
BrotliHasherParams params;
@@ -73,13 +76,13 @@ typedef struct HasherSearchResult {
* There is no effort to ensure that it is a prime, the oddity is enough
for this use.
* The number has been tuned heuristically against compression benchmarks. */
static const uint32_t kHashMul32 = 0x1e35a7bd;
static const uint64_t kHashMul64 = BROTLI_MAKE_UINT64_T(0x1e35a7bd, 0x1e35a7bd);
static const uint32_t kHashMul32 = 0x1E35A7BD;
static const uint64_t kHashMul64 = BROTLI_MAKE_UINT64_T(0x1E35A7BD, 0x1E35A7BD);
static const uint64_t kHashMul64Long =
BROTLI_MAKE_UINT64_T(0x1fe35a7bU, 0xd3579bd3U);
BROTLI_MAKE_UINT64_T(0x1FE35A7Bu, 0xD3579BD3u);
static BROTLI_INLINE uint32_t Hash14(const uint8_t* data) {
uint32_t h = BROTLI_UNALIGNED_LOAD32(data) * kHashMul32;
uint32_t h = BROTLI_UNALIGNED_LOAD32LE(data) * kHashMul32;
/* The higher bits contain more mixture from the multiplication,
so we take our results from there. */
return h >> (32 - 14);
@@ -146,34 +149,35 @@ static BROTLI_INLINE score_t BackwardReferencePenaltyUsingLastDistance(
}
static BROTLI_INLINE BROTLI_BOOL TestStaticDictionaryItem(
const BrotliDictionary* dictionary, size_t item, const uint8_t* data,
size_t max_length, size_t max_backward, HasherSearchResult* out) {
const BrotliEncoderDictionary* dictionary, size_t item, const uint8_t* data,
size_t max_length, size_t max_backward, size_t max_distance,
HasherSearchResult* out) {
size_t len;
size_t dist;
size_t word_idx;
size_t offset;
size_t matchlen;
size_t backward;
score_t score;
len = item & 0x1F;
dist = item >> 5;
offset = dictionary->offsets_by_length[len] + len * dist;
word_idx = item >> 5;
offset = dictionary->words->offsets_by_length[len] + len * word_idx;
if (len > max_length) {
return BROTLI_FALSE;
}
matchlen =
FindMatchLengthWithLimit(data, &dictionary->data[offset], len);
if (matchlen + kCutoffTransformsCount <= len || matchlen == 0) {
FindMatchLengthWithLimit(data, &dictionary->words->data[offset], len);
if (matchlen + dictionary->cutoffTransformsCount <= len || matchlen == 0) {
return BROTLI_FALSE;
}
{
size_t cut = len - matchlen;
size_t transform_id =
(cut << 2) + (size_t)((kCutoffTransforms >> (cut * 6)) & 0x3F);
backward = max_backward + dist + 1 +
(transform_id << dictionary->size_bits_by_length[len]);
size_t transform_id = (cut << 2) +
(size_t)((dictionary->cutoffTransforms >> (cut * 6)) & 0x3F);
backward = max_backward + 1 + word_idx +
(transform_id << dictionary->words->size_bits_by_length[len]);
}
if (backward >= BROTLI_MAX_DISTANCE) {
if (backward > max_distance) {
return BROTLI_FALSE;
}
score = BackwardReferenceScore(matchlen, backward);
@@ -188,9 +192,10 @@ static BROTLI_INLINE BROTLI_BOOL TestStaticDictionaryItem(
}
static BROTLI_INLINE void SearchInStaticDictionary(
const BrotliDictionary* dictionary, const uint16_t* dictionary_hash,
const BrotliEncoderDictionary* dictionary,
HasherHandle handle, const uint8_t* data, size_t max_length,
size_t max_backward, HasherSearchResult* out, BROTLI_BOOL shallow) {
size_t max_backward, size_t max_distance,
HasherSearchResult* out, BROTLI_BOOL shallow) {
size_t key;
size_t i;
HasherCommon* self = GetHasherCommon(handle);
@@ -199,11 +204,11 @@ static BROTLI_INLINE void SearchInStaticDictionary(
}
key = Hash14(data) << 1;
for (i = 0; i < (shallow ? 1u : 2u); ++i, ++key) {
size_t item = dictionary_hash[key];
size_t item = dictionary->hash_table[key];
self->dict_num_lookups++;
if (item != 0) {
BROTLI_BOOL item_matches = TestStaticDictionaryItem(
dictionary, item, data, max_length, max_backward, out);
dictionary, item, data, max_length, max_backward, max_distance, out);
if (item_matches) {
self->dict_num_matches++;
}
@@ -338,11 +343,57 @@ static BROTLI_INLINE size_t BackwardMatchLengthCode(const BackwardMatch* self) {
#undef BUCKET_BITS
#undef HASHER
/* fast large window hashers */
#define HASHER() HROLLING_FAST
#define CHUNKLEN 32
#define JUMP 4
#define NUMBUCKETS 16777216
#define MASK ((NUMBUCKETS * 64) - 1)
#include "./hash_rolling_inc.h" /* NOLINT(build/include) */
#undef JUMP
#undef HASHER
#define HASHER() HROLLING
#define JUMP 1
#include "./hash_rolling_inc.h" /* NOLINT(build/include) */
#undef MASK
#undef NUMBUCKETS
#undef JUMP
#undef CHUNKLEN
#undef HASHER
#define HASHER() H35
#define HASHER_A H3
#define HASHER_B HROLLING_FAST
#include "./hash_composite_inc.h" /* NOLINT(build/include) */
#undef HASHER_A
#undef HASHER_B
#undef HASHER
#define HASHER() H55
#define HASHER_A H54
#define HASHER_B HROLLING_FAST
#include "./hash_composite_inc.h" /* NOLINT(build/include) */
#undef HASHER_A
#undef HASHER_B
#undef HASHER
#define HASHER() H65
#define HASHER_A H6
#define HASHER_B HROLLING
#include "./hash_composite_inc.h" /* NOLINT(build/include) */
#undef HASHER_A
#undef HASHER_B
#undef HASHER
#undef FN
#undef CAT
#undef EXPAND_CAT
#define FOR_GENERIC_HASHERS(H) H(2) H(3) H(4) H(5) H(6) H(40) H(41) H(42) H(54)
#define FOR_GENERIC_HASHERS(H) H(2) H(3) H(4) H(5) H(6) H(40) H(41) H(42) H(54)\
H(35) H(55) H(65)
#define FOR_ALL_HASHERS(H) FOR_GENERIC_HASHERS(H) H(10)
static BROTLI_INLINE void DestroyHasher(

133
C/brotli/enc/hash_composite_inc.h Normal file
View File

@@ -0,0 +1,133 @@
/* NOLINT(build/header_guard) */
/* Copyright 2018 Google Inc. All Rights Reserved.
Distributed under MIT license.
See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
*/
/* template parameters: FN, HASHER_A, HASHER_B */
/* Composite hasher: This hasher allows to combine two other hashers, HASHER_A
and HASHER_B. */
#define HashComposite HASHER()
#define FN_A(X) EXPAND_CAT(X, HASHER_A)
#define FN_B(X) EXPAND_CAT(X, HASHER_B)
static BROTLI_INLINE size_t FN(HashTypeLength)(void) {
size_t a = FN_A(HashTypeLength)();
size_t b = FN_B(HashTypeLength)();
return a > b ? a : b;
}
static BROTLI_INLINE size_t FN(StoreLookahead)(void) {
size_t a = FN_A(StoreLookahead)();
size_t b = FN_B(StoreLookahead)();
return a > b ? a : b;
}
typedef struct HashComposite {
HasherHandle ha;
HasherHandle hb;
const BrotliEncoderParams* params;
} HashComposite;
static BROTLI_INLINE HashComposite* FN(Self)(HasherHandle handle) {
return (HashComposite*)&(GetHasherCommon(handle)[1]);
}
static void FN(Initialize)(
HasherHandle handle, const BrotliEncoderParams* params) {
HashComposite* self = FN(Self)(handle);
self->ha = 0;
self->hb = 0;
self->params = params;
/* TODO: Initialize of the hashers is defered to Prepare (and params
remembered here) because we don't get the one_shot and input_size params
here that are needed to know the memory size of them. Instead provide
those params to all hashers FN(Initialize) */
}
static void FN(Prepare)(HasherHandle handle, BROTLI_BOOL one_shot,
size_t input_size, const uint8_t* data) {
HashComposite* self = FN(Self)(handle);
if (!self->ha) {
HasherCommon* common_a;
HasherCommon* common_b;
self->ha = handle + sizeof(HasherCommon) + sizeof(HashComposite);
common_a = (HasherCommon*)self->ha;
common_a->params = self->params->hasher;
common_a->is_prepared_ = BROTLI_FALSE;
common_a->dict_num_lookups = 0;
common_a->dict_num_matches = 0;
FN_A(Initialize)(self->ha, self->params);
self->hb = self->ha + sizeof(HasherCommon) + FN_A(HashMemAllocInBytes)(
self->params, one_shot, input_size);
common_b = (HasherCommon*)self->hb;
common_b->params = self->params->hasher;
common_b->is_prepared_ = BROTLI_FALSE;
common_b->dict_num_lookups = 0;
common_b->dict_num_matches = 0;
FN_B(Initialize)(self->hb, self->params);
}
FN_A(Prepare)(self->ha, one_shot, input_size, data);
FN_B(Prepare)(self->hb, one_shot, input_size, data);
}
static BROTLI_INLINE size_t FN(HashMemAllocInBytes)(
const BrotliEncoderParams* params, BROTLI_BOOL one_shot,
size_t input_size) {
return sizeof(HashComposite) + 2 * sizeof(HasherCommon) +
FN_A(HashMemAllocInBytes)(params, one_shot, input_size) +
FN_B(HashMemAllocInBytes)(params, one_shot, input_size);
}
static BROTLI_INLINE void FN(Store)(HasherHandle BROTLI_RESTRICT handle,
const uint8_t* BROTLI_RESTRICT data, const size_t mask, const size_t ix) {
HashComposite* self = FN(Self)(handle);
FN_A(Store)(self->ha, data, mask, ix);
FN_B(Store)(self->hb, data, mask, ix);
}
static BROTLI_INLINE void FN(StoreRange)(HasherHandle handle,
const uint8_t* data, const size_t mask, const size_t ix_start,
const size_t ix_end) {
HashComposite* self = FN(Self)(handle);
FN_A(StoreRange)(self->ha, data, mask, ix_start, ix_end);
FN_B(StoreRange)(self->hb, data, mask, ix_start, ix_end);
}
static BROTLI_INLINE void FN(StitchToPreviousBlock)(HasherHandle handle,
size_t num_bytes, size_t position, const uint8_t* ringbuffer,
size_t ring_buffer_mask) {
HashComposite* self = FN(Self)(handle);
FN_A(StitchToPreviousBlock)(self->ha, num_bytes, position, ringbuffer,
ring_buffer_mask);
FN_B(StitchToPreviousBlock)(self->hb, num_bytes, position, ringbuffer,
ring_buffer_mask);
}
static BROTLI_INLINE void FN(PrepareDistanceCache)(
HasherHandle handle, int* BROTLI_RESTRICT distance_cache) {
HashComposite* self = FN(Self)(handle);
FN_A(PrepareDistanceCache)(self->ha, distance_cache);
FN_B(PrepareDistanceCache)(self->hb, distance_cache);
}
static BROTLI_INLINE void FN(FindLongestMatch)(HasherHandle handle,
const BrotliEncoderDictionary* dictionary,
const uint8_t* BROTLI_RESTRICT data, const size_t ring_buffer_mask,
const int* BROTLI_RESTRICT distance_cache, const size_t cur_ix,
const size_t max_length, const size_t max_backward, const size_t gap,
const size_t max_distance, HasherSearchResult* BROTLI_RESTRICT out) {
HashComposite* self = FN(Self)(handle);
FN_A(FindLongestMatch)(self->ha, dictionary, data, ring_buffer_mask,
distance_cache, cur_ix, max_length, max_backward, gap, max_distance, out);
FN_B(FindLongestMatch)(self->hb, dictionary, data, ring_buffer_mask,
distance_cache, cur_ix, max_length, max_backward, gap, max_distance, out);
}
#undef HashComposite

17
C/brotli/enc/hash_forgetful_chain_inc.h
View File

@@ -28,8 +28,8 @@ static BROTLI_INLINE size_t FN(HashTypeLength)(void) { return 4; }
static BROTLI_INLINE size_t FN(StoreLookahead)(void) { return 4; }
/* HashBytes is the function that chooses the bucket to place the address in.*/
static BROTLI_INLINE size_t FN(HashBytes)(const uint8_t *data) {
const uint32_t h = BROTLI_UNALIGNED_LOAD32(data) * kHashMul32;
static BROTLI_INLINE size_t FN(HashBytes)(const uint8_t* data) {
const uint32_t h = BROTLI_UNALIGNED_LOAD32LE(data) * kHashMul32;
/* The higher bits contain more mixture from the multiplication,
so we take our results from there. */
return h >> (32 - BUCKET_BITS);
@@ -115,7 +115,7 @@ static BROTLI_INLINE void FN(Store)(HasherHandle BROTLI_RESTRICT handle,
}
static BROTLI_INLINE void FN(StoreRange)(HasherHandle handle,
const uint8_t *data, const size_t mask, const size_t ix_start,
const uint8_t* data, const size_t mask, const size_t ix_start,
const size_t ix_end) {
size_t i;
for (i = ix_start; i < ix_end; ++i) {
@@ -154,11 +154,12 @@ static BROTLI_INLINE void FN(PrepareDistanceCache)(
Writes the best match into |out|.
|out|->score is updated only if a better match is found. */
static BROTLI_INLINE void FN(FindLongestMatch)(HasherHandle handle,
const BrotliDictionary* dictionary, const uint16_t* dictionary_hash,
const BrotliEncoderDictionary* dictionary,
const uint8_t* BROTLI_RESTRICT data, const size_t ring_buffer_mask,
const int* BROTLI_RESTRICT distance_cache,
const size_t cur_ix, const size_t max_length, const size_t max_backward,
const size_t gap, HasherSearchResult* BROTLI_RESTRICT out) {
const size_t gap, const size_t max_distance,
HasherSearchResult* BROTLI_RESTRICT out) {
HashForgetfulChain* self = FN(Self)(handle);
const size_t cur_ix_masked = cur_ix & ring_buffer_mask;
/* Don't accept a short copy from far away. */
@@ -240,9 +241,9 @@ static BROTLI_INLINE void FN(FindLongestMatch)(HasherHandle handle,
FN(Store)(handle, data, ring_buffer_mask, cur_ix);
}
if (out->score == min_score) {
SearchInStaticDictionary(dictionary, dictionary_hash,
handle, &data[cur_ix_masked], max_length, max_backward + gap, out,
BROTLI_FALSE);
SearchInStaticDictionary(dictionary,
handle, &data[cur_ix_masked], max_length, max_backward + gap,
max_distance, out, BROTLI_FALSE);
}
}

16
C/brotli/enc/hash_longest_match64_inc.h
View File

@@ -20,7 +20,7 @@ static BROTLI_INLINE size_t FN(HashTypeLength)(void) { return 8; }
static BROTLI_INLINE size_t FN(StoreLookahead)(void) { return 8; }
/* HashBytes is the function that chooses the bucket to place the address in. */
static BROTLI_INLINE uint32_t FN(HashBytes)(const uint8_t *data,
static BROTLI_INLINE uint32_t FN(HashBytes)(const uint8_t* data,
const uint64_t mask,
const int shift) {
const uint64_t h = (BROTLI_UNALIGNED_LOAD64LE(data) & mask) * kHashMul64Long;
@@ -105,7 +105,7 @@ static BROTLI_INLINE size_t FN(HashMemAllocInBytes)(
/* Look at 4 bytes at &data[ix & mask].
Compute a hash from these, and store the value of ix at that position. */
static BROTLI_INLINE void FN(Store)(HasherHandle handle, const uint8_t *data,
static BROTLI_INLINE void FN(Store)(HasherHandle handle, const uint8_t* data,
const size_t mask, const size_t ix) {
HashLongestMatch* self = FN(Self)(handle);
uint16_t* num = FN(Num)(self);
@@ -119,7 +119,7 @@ static BROTLI_INLINE void FN(Store)(HasherHandle handle, const uint8_t *data,
}
static BROTLI_INLINE void FN(StoreRange)(HasherHandle handle,
const uint8_t *data, const size_t mask, const size_t ix_start,
const uint8_t* data, const size_t mask, const size_t ix_start,
const size_t ix_end) {
size_t i;
for (i = ix_start; i < ix_end; ++i) {
@@ -158,11 +158,11 @@ static BROTLI_INLINE void FN(PrepareDistanceCache)(
Writes the best match into |out|.
|out|->score is updated only if a better match is found. */
static BROTLI_INLINE void FN(FindLongestMatch)(HasherHandle handle,
const BrotliDictionary* dictionary, const uint16_t* dictionary_hash,
const BrotliEncoderDictionary* dictionary,
const uint8_t* BROTLI_RESTRICT data, const size_t ring_buffer_mask,
const int* BROTLI_RESTRICT distance_cache, const size_t cur_ix,
const size_t max_length, const size_t max_backward, const size_t gap,
HasherSearchResult* BROTLI_RESTRICT out) {
const size_t max_distance, HasherSearchResult* BROTLI_RESTRICT out) {
HasherCommon* common = GetHasherCommon(handle);
HashLongestMatch* self = FN(Self)(handle);
uint16_t* num = FN(Num)(self);
@@ -257,9 +257,9 @@ static BROTLI_INLINE void FN(FindLongestMatch)(HasherHandle handle,
++num[key];
}
if (min_score == out->score) {
SearchInStaticDictionary(dictionary, dictionary_hash,
handle, &data[cur_ix_masked], max_length, max_backward + gap, out,
BROTLI_FALSE);
SearchInStaticDictionary(dictionary,
handle, &data[cur_ix_masked], max_length, max_backward + gap,
max_distance, out, BROTLI_FALSE);
}
}

16
C/brotli/enc/hash_longest_match_inc.h
View File

@@ -20,8 +20,8 @@ static BROTLI_INLINE size_t FN(HashTypeLength)(void) { return 4; }
static BROTLI_INLINE size_t FN(StoreLookahead)(void) { return 4; }
/* HashBytes is the function that chooses the bucket to place the address in. */
static uint32_t FN(HashBytes)(const uint8_t *data, const int shift) {
uint32_t h = BROTLI_UNALIGNED_LOAD32(data) * kHashMul32;
static uint32_t FN(HashBytes)(const uint8_t* data, const int shift) {
uint32_t h = BROTLI_UNALIGNED_LOAD32LE(data) * kHashMul32;
/* The higher bits contain more mixture from the multiplication,
so we take our results from there. */
return (uint32_t)(h >> shift);
@@ -112,7 +112,7 @@ static BROTLI_INLINE void FN(Store)(HasherHandle handle, const uint8_t* data,
}
static BROTLI_INLINE void FN(StoreRange)(HasherHandle handle,
const uint8_t *data, const size_t mask, const size_t ix_start,
const uint8_t* data, const size_t mask, const size_t ix_start,
const size_t ix_end) {
size_t i;
for (i = ix_start; i < ix_end; ++i) {
@@ -151,11 +151,11 @@ static BROTLI_INLINE void FN(PrepareDistanceCache)(
Writes the best match into |out|.
|out|->score is updated only if a better match is found. */
static BROTLI_INLINE void FN(FindLongestMatch)(HasherHandle handle,
const BrotliDictionary* dictionary, const uint16_t* dictionary_hash,
const BrotliEncoderDictionary* dictionary,
const uint8_t* BROTLI_RESTRICT data, const size_t ring_buffer_mask,
const int* BROTLI_RESTRICT distance_cache, const size_t cur_ix,
const size_t max_length, const size_t max_backward, const size_t gap,
HasherSearchResult* BROTLI_RESTRICT out) {
const size_t max_distance, HasherSearchResult* BROTLI_RESTRICT out) {
HasherCommon* common = GetHasherCommon(handle);
HashLongestMatch* self = FN(Self)(handle);
uint16_t* num = FN(Num)(self);
@@ -249,9 +249,9 @@ static BROTLI_INLINE void FN(FindLongestMatch)(HasherHandle handle,
++num[key];
}
if (min_score == out->score) {
SearchInStaticDictionary(dictionary, dictionary_hash,
handle, &data[cur_ix_masked], max_length, max_backward + gap, out,
BROTLI_FALSE);
SearchInStaticDictionary(dictionary,
handle, &data[cur_ix_masked], max_length, max_backward + gap,
max_distance, out, BROTLI_FALSE);
}
}

19
C/brotli/enc/hash_longest_match_quickly_inc.h
View File

@@ -81,7 +81,7 @@ static BROTLI_INLINE size_t FN(HashMemAllocInBytes)(
Compute a hash from these, and store the value somewhere within
[ix .. ix+3]. */
static BROTLI_INLINE void FN(Store)(HasherHandle handle,
const uint8_t *data, const size_t mask, const size_t ix) {
const uint8_t* data, const size_t mask, const size_t ix) {
const uint32_t key = FN(HashBytes)(&data[ix & mask]);
/* Wiggle the value with the bucket sweep range. */
const uint32_t off = (ix >> 3) % BUCKET_SWEEP;
@@ -89,7 +89,7 @@ static BROTLI_INLINE void FN(Store)(HasherHandle handle,
}
static BROTLI_INLINE void FN(StoreRange)(HasherHandle handle,
const uint8_t *data, const size_t mask, const size_t ix_start,
const uint8_t* data, const size_t mask, const size_t ix_start,
const size_t ix_end) {
size_t i;
for (i = ix_start; i < ix_end; ++i) {
@@ -125,11 +125,12 @@ static BROTLI_INLINE void FN(PrepareDistanceCache)(
Writes the best match into |out|.
|out|->score is updated only if a better match is found. */
static BROTLI_INLINE void FN(FindLongestMatch)(
HasherHandle handle, const BrotliDictionary* dictionary,
const uint16_t* dictionary_hash, const uint8_t* BROTLI_RESTRICT data,
HasherHandle handle, const BrotliEncoderDictionary* dictionary,
const uint8_t* BROTLI_RESTRICT data,
const size_t ring_buffer_mask, const int* BROTLI_RESTRICT distance_cache,
const size_t cur_ix, const size_t max_length, const size_t max_backward,
const size_t gap, HasherSearchResult* BROTLI_RESTRICT out) {
const size_t gap, const size_t max_distance,
HasherSearchResult* BROTLI_RESTRICT out) {
HashLongestMatchQuickly* self = FN(Self)(handle);
const size_t best_len_in = out->len;
const size_t cur_ix_masked = cur_ix & ring_buffer_mask;
@@ -191,7 +192,7 @@ static BROTLI_INLINE void FN(FindLongestMatch)(
}
}
} else {
uint32_t *bucket = self->buckets_ + key;
uint32_t* bucket = self->buckets_ + key;
int i;
prev_ix = *bucket++;
for (i = 0; i < BUCKET_SWEEP; ++i, prev_ix = *bucket++) {
@@ -221,9 +222,9 @@ static BROTLI_INLINE void FN(FindLongestMatch)(
}
}
if (USE_DICTIONARY && min_score == out->score) {
SearchInStaticDictionary(dictionary, dictionary_hash,
handle, &data[cur_ix_masked], max_length, max_backward + gap, out,
BROTLI_TRUE);
SearchInStaticDictionary(dictionary,
handle, &data[cur_ix_masked], max_length, max_backward + gap,
max_distance, out, BROTLI_TRUE);
}
self->buckets_[key + ((cur_ix >> 3) % BUCKET_SWEEP)] = (uint32_t)cur_ix;
}

215
C/brotli/enc/hash_rolling_inc.h Normal file
View File

@@ -0,0 +1,215 @@
/* NOLINT(build/header_guard) */
/* Copyright 2018 Google Inc. All Rights Reserved.
Distributed under MIT license.
See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
*/
/* template parameters: FN, JUMP, NUMBUCKETS, MASK, CHUNKLEN */
/* NUMBUCKETS / (MASK + 1) = probability of storing and using hash code. */
/* JUMP = skip bytes for speedup */
/* Rolling hash for long distance long string matches. Stores one position
per bucket, bucket key is computed over a long region. */
#define HashRolling HASHER()
static const uint32_t FN(kRollingHashMul32) = 69069;
static const uint32_t FN(kInvalidPos) = 0xffffffff;
/* This hasher uses a longer forward length, but returning a higher value here
will hurt compression by the main hasher when combined with a composite
hasher. The hasher tests for forward itself instead. */
static BROTLI_INLINE size_t FN(HashTypeLength)(void) { return 4; }
static BROTLI_INLINE size_t FN(StoreLookahead)(void) { return 4; }
/* Computes a code from a single byte. A lookup table of 256 values could be
used, but simply adding 1 works about as good. */
static uint32_t FN(HashByte)(uint8_t byte) {
return (uint32_t)byte + 1u;
}
static uint32_t FN(HashRollingFunctionInitial)(uint32_t state, uint8_t add,
uint32_t factor) {
return (uint32_t)(factor * state + FN(HashByte)(add));
}
static uint32_t FN(HashRollingFunction)(uint32_t state, uint8_t add,
uint8_t rem, uint32_t factor,
uint32_t factor_remove) {
return (uint32_t)(factor * state +
FN(HashByte)(add) - factor_remove * FN(HashByte)(rem));
}
typedef struct HashRolling {
uint32_t state;
uint32_t* table;
size_t next_ix;
uint32_t chunk_len;
uint32_t factor;
uint32_t factor_remove;
} HashRolling;
static BROTLI_INLINE HashRolling* FN(Self)(HasherHandle handle) {
return (HashRolling*)&(GetHasherCommon(handle)[1]);
}
static void FN(Initialize)(
HasherHandle handle, const BrotliEncoderParams* params) {
HashRolling* self = FN(Self)(handle);
size_t i;
self->state = 0;
self->next_ix = 0;
self->factor = FN(kRollingHashMul32);
/* Compute the factor of the oldest byte to remove: factor**steps modulo
0xffffffff (the multiplications rely on 32-bit overflow) */
self->factor_remove = 1;
for (i = 0; i < CHUNKLEN; i += JUMP) {
self->factor_remove *= self->factor;
}
self->table = (uint32_t*)((HasherHandle)self + sizeof(HashRolling));
for (i = 0; i < NUMBUCKETS; i++) {
self->table[i] = FN(kInvalidPos);
}
BROTLI_UNUSED(params);
}
static void FN(Prepare)(HasherHandle handle, BROTLI_BOOL one_shot,
size_t input_size, const uint8_t* data) {
HashRolling* self = FN(Self)(handle);
size_t i;
/* Too small size, cannot use this hasher. */
if (input_size < CHUNKLEN) return;
self->state = 0;
for (i = 0; i < CHUNKLEN; i += JUMP) {
self->state = FN(HashRollingFunctionInitial)(
self->state, data[i], self->factor);
}
BROTLI_UNUSED(one_shot);
}
static BROTLI_INLINE size_t FN(HashMemAllocInBytes)(
const BrotliEncoderParams* params, BROTLI_BOOL one_shot,
size_t input_size) {
return sizeof(HashRolling) + NUMBUCKETS * sizeof(uint32_t);
BROTLI_UNUSED(params);
BROTLI_UNUSED(one_shot);
BROTLI_UNUSED(input_size);
}
static BROTLI_INLINE void FN(Store)(HasherHandle BROTLI_RESTRICT handle,
const uint8_t* BROTLI_RESTRICT data, const size_t mask, const size_t ix) {
BROTLI_UNUSED(handle);
BROTLI_UNUSED(data);
BROTLI_UNUSED(mask);
BROTLI_UNUSED(ix);
}
static BROTLI_INLINE void FN(StoreRange)(HasherHandle handle,
const uint8_t* data, const size_t mask, const size_t ix_start,
const size_t ix_end) {
BROTLI_UNUSED(handle);
BROTLI_UNUSED(data);
BROTLI_UNUSED(mask);
BROTLI_UNUSED(ix_start);
BROTLI_UNUSED(ix_end);
}
static BROTLI_INLINE void FN(StitchToPreviousBlock)(HasherHandle handle,
size_t num_bytes, size_t position, const uint8_t* ringbuffer,
size_t ring_buffer_mask) {
/* In this case we must re-initialize the hasher from scratch from the
current position. */
HashRolling* self = FN(Self)(handle);
size_t position_masked;
size_t available = num_bytes;
if ((position & (JUMP - 1)) != 0) {
size_t diff = JUMP - (position & (JUMP - 1));
available = (diff > available) ? 0 : (available - diff);
position += diff;
}
position_masked = position & ring_buffer_mask;
/* wrapping around ringbuffer not handled. */
if (available > ring_buffer_mask - position_masked) {
available = ring_buffer_mask - position_masked;
}
FN(Prepare)(handle, BROTLI_FALSE, available,
ringbuffer + (position & ring_buffer_mask));
self->next_ix = position;
BROTLI_UNUSED(num_bytes);
}
static BROTLI_INLINE void FN(PrepareDistanceCache)(
HasherHandle handle, int* BROTLI_RESTRICT distance_cache) {
BROTLI_UNUSED(handle);
BROTLI_UNUSED(distance_cache);
}
static BROTLI_INLINE void FN(FindLongestMatch)(HasherHandle handle,
const BrotliEncoderDictionary* dictionary,
const uint8_t* BROTLI_RESTRICT data, const size_t ring_buffer_mask,
const int* BROTLI_RESTRICT distance_cache, const size_t cur_ix,
const size_t max_length, const size_t max_backward, const size_t gap,
const size_t max_distance, HasherSearchResult* BROTLI_RESTRICT out) {
HashRolling* self = FN(Self)(handle);
const size_t cur_ix_masked = cur_ix & ring_buffer_mask;
size_t pos = self->next_ix;
if ((cur_ix & (JUMP - 1)) != 0) return;
/* Not enough lookahead */
if (max_length < CHUNKLEN) return;
for (pos = self->next_ix; pos <= cur_ix; pos += JUMP) {
uint32_t code = self->state & MASK;
uint8_t rem = data[pos & ring_buffer_mask];
uint8_t add = data[(pos + CHUNKLEN) & ring_buffer_mask];
size_t found_ix = FN(kInvalidPos);
self->state = FN(HashRollingFunction)(
self->state, add, rem, self->factor, self->factor_remove);
if (code < NUMBUCKETS) {
found_ix = self->table[code];
self->table[code] = (uint32_t)pos;
if (pos == cur_ix && found_ix != FN(kInvalidPos)) {
/* The cast to 32-bit makes backward distances up to 4GB work even
if cur_ix is above 4GB, despite using 32-bit values in the table. */
size_t backward = (uint32_t)(cur_ix - found_ix);
if (backward <= max_backward) {
const size_t found_ix_masked = found_ix & ring_buffer_mask;
const size_t len = FindMatchLengthWithLimit(&data[found_ix_masked],
&data[cur_ix_masked],
max_length);
if (len >= 4 && len > out->len) {
score_t score = BackwardReferenceScore(len, backward);
if (score > out->score) {
out->len = len;
out->distance = backward;
out->score = score;
out->len_code_delta = 0;
}
}
}
}
}
}
self->next_ix = cur_ix + JUMP;
/* NOTE: this hasher does not search in the dictionary. It is used as
backup-hasher, the main hasher already searches in it. */
BROTLI_UNUSED(dictionary);
BROTLI_UNUSED(distance_cache);
BROTLI_UNUSED(gap);
BROTLI_UNUSED(max_distance);
}
#undef HashRolling

17
C/brotli/enc/hash_to_binary_tree_inc.h
View File

@@ -24,8 +24,8 @@ static BROTLI_INLINE size_t FN(StoreLookahead)(void) {
return MAX_TREE_COMP_LENGTH;
}
static uint32_t FN(HashBytes)(const uint8_t *data) {
uint32_t h = BROTLI_UNALIGNED_LOAD32(data) * kHashMul32;
static uint32_t FN(HashBytes)(const uint8_t* data) {
uint32_t h = BROTLI_UNALIGNED_LOAD32LE(data) * kHashMul32;
/* The higher bits contain more mixture from the multiplication,
so we take our results from there. */
return h >> (32 - BUCKET_BITS);
@@ -154,12 +154,13 @@ static BROTLI_INLINE BackwardMatch* FN(StoreAndFindMatches)(
{
const size_t cur_len = BROTLI_MIN(size_t, best_len_left, best_len_right);
size_t len;
assert(cur_len <= MAX_TREE_COMP_LENGTH);
BROTLI_DCHECK(cur_len <= MAX_TREE_COMP_LENGTH);
len = cur_len +
FindMatchLengthWithLimit(&data[cur_ix_masked + cur_len],
&data[prev_ix_masked + cur_len],
max_length - cur_len);
assert(0 == memcmp(&data[cur_ix_masked], &data[prev_ix_masked], len));
BROTLI_DCHECK(
0 == memcmp(&data[cur_ix_masked], &data[prev_ix_masked], len));
if (matches && len > *best_len) {
*best_len = len;
InitBackwardMatch(matches++, backward, len);
@@ -199,7 +200,7 @@ static BROTLI_INLINE BackwardMatch* FN(StoreAndFindMatches)(
sorted by strictly increasing length and (non-strictly) increasing
distance. */
static BROTLI_INLINE size_t FN(FindAllMatches)(HasherHandle handle,
const BrotliDictionary* dictionary, const uint8_t* data,
const BrotliEncoderDictionary* dictionary, const uint8_t* data,
const size_t ring_buffer_mask, const size_t cur_ix,
const size_t max_length, const size_t max_backward, const size_t gap,
const BrotliEncoderParams* params, BackwardMatch* matches) {
@@ -251,7 +252,7 @@ static BROTLI_INLINE size_t FN(FindAllMatches)(HasherHandle handle,
uint32_t dict_id = dict_matches[l];
if (dict_id < kInvalidMatch) {
size_t distance = max_backward + gap + (dict_id >> 5) + 1;
if (distance < BROTLI_MAX_DISTANCE) {
if (distance <= params->dist.max_distance) {
InitDictionaryBackwardMatch(matches++, distance, l, dict_id & 31);
}
}
@@ -264,7 +265,7 @@ static BROTLI_INLINE size_t FN(FindAllMatches)(HasherHandle handle,
/* Stores the hash of the next 4 bytes and re-roots the binary tree at the
current sequence, without returning any matches.
REQUIRES: ix + MAX_TREE_COMP_LENGTH <= end-of-current-block */
static BROTLI_INLINE void FN(Store)(HasherHandle handle, const uint8_t *data,
static BROTLI_INLINE void FN(Store)(HasherHandle handle, const uint8_t* data,
const size_t mask, const size_t ix) {
HashToBinaryTree* self = FN(Self)(handle);
/* Maximum distance is window size - 16, see section 9.1. of the spec. */
@@ -274,7 +275,7 @@ static BROTLI_INLINE void FN(Store)(HasherHandle handle, const uint8_t *data,
}
static BROTLI_INLINE void FN(StoreRange)(HasherHandle handle,
const uint8_t *data, const size_t mask, const size_t ix_start,
const uint8_t* data, const size_t mask, const size_t ix_start,
const size_t ix_end) {
size_t i = ix_start;
size_t j = ix_start;

9
C/brotli/enc/histogram.h
View File

@@ -12,16 +12,19 @@
#include <string.h> /* memset */
#include "../common/constants.h"
#include "../common/context.h"
#include "../common/platform.h"
#include "../types.h"
#include "./block_splitter.h"
#include "./command.h"
#include "./context.h"
#include "./port.h"
#if defined(__cplusplus) || defined(c_plusplus)
extern "C" {
#endif
/* The distance symbols effectively used by "Large Window Brotli" (32-bit). */
#define BROTLI_NUM_HISTOGRAM_DISTANCE_SYMBOLS 544
#define FN(X) X ## Literal
#define DATA_SIZE BROTLI_NUM_LITERAL_SYMBOLS
#define DataType uint8_t
@@ -38,7 +41,7 @@ extern "C" {
#undef FN
#define FN(X) X ## Distance
#define DATA_SIZE BROTLI_NUM_DISTANCE_SYMBOLS
#define DATA_SIZE BROTLI_NUM_HISTOGRAM_DISTANCE_SYMBOLS
#include "./histogram_inc.h" /* NOLINT(build/include) */
#undef DataType
#undef DATA_SIZE

2
C/brotli/enc/histogram_inc.h
View File

@@ -33,7 +33,7 @@ static BROTLI_INLINE void FN(HistogramAdd)(FN(Histogram)* self, size_t val) {
}
static BROTLI_INLINE void FN(HistogramAddVector)(FN(Histogram)* self,
const DataType *p, size_t n) {
const DataType* p, size_t n) {
self->total_count_ += n;
n += 1;
while (--n) ++self->data_[*p++];

4
C/brotli/enc/literal_cost.h
View File

@@ -10,8 +10,8 @@
#ifndef BROTLI_ENC_LITERAL_COST_H_
#define BROTLI_ENC_LITERAL_COST_H_
#include "../common/platform.h"
#include "../types.h"
#include "./port.h"
#if defined(__cplusplus) || defined(c_plusplus)
extern "C" {
@@ -21,7 +21,7 @@ extern "C" {
ring-buffer (data, mask) will take entropy coded and writes these estimates
to the cost[0..len) array. */
BROTLI_INTERNAL void BrotliEstimateBitCostsForLiterals(
size_t pos, size_t len, size_t mask, const uint8_t *data, float *cost);
size_t pos, size_t len, size_t mask, const uint8_t* data, float* cost);
#if defined(__cplusplus) || defined(c_plusplus)
} /* extern "C" */

41
C/brotli/enc/memory.h
View File

@@ -9,8 +9,10 @@
#ifndef BROTLI_ENC_MEMORY_H_
#define BROTLI_ENC_MEMORY_H_
#include <string.h> /* memcpy */
#include "../common/platform.h"
#include "../types.h"
#include "./port.h"
#if defined(__cplusplus) || defined(c_plusplus)
extern "C" {
@@ -56,6 +58,43 @@ BROTLI_INTERNAL void BrotliFree(MemoryManager* m, void* p);
BROTLI_INTERNAL void BrotliWipeOutMemoryManager(MemoryManager* m);
/*
Dynamically grows array capacity to at least the requested size
M: MemoryManager
T: data type
A: array
C: capacity
R: requested size
*/
#define BROTLI_ENSURE_CAPACITY(M, T, A, C, R) { \
if (C < (R)) { \
size_t _new_size = (C == 0) ? (R) : C; \
T* new_array; \
while (_new_size < (R)) _new_size *= 2; \
new_array = BROTLI_ALLOC((M), T, _new_size); \
if (!BROTLI_IS_OOM(M) && C != 0) \
memcpy(new_array, A, C * sizeof(T)); \
BROTLI_FREE((M), A); \
A = new_array; \
C = _new_size; \
} \
}
/*
Appends value and dynamically grows array capacity when needed
M: MemoryManager
T: data type
A: array
C: array capacity
S: array size
V: value to append
*/
#define BROTLI_ENSURE_CAPACITY_APPEND(M, T, A, C, S, V) { \
(S)++; \
BROTLI_ENSURE_CAPACITY(M, T, A, C, S); \
A[(S) - 1] = (V); \
}
#if defined(__cplusplus) || defined(c_plusplus)
} /* extern "C" */
#endif

21
C/brotli/enc/metablock.h
View File

@@ -10,13 +10,13 @@
#ifndef BROTLI_ENC_METABLOCK_H_
#define BROTLI_ENC_METABLOCK_H_
#include "../common/context.h"
#include "../common/platform.h"
#include "../types.h"
#include "./block_splitter.h"
#include "./command.h"
#include "./context.h"
#include "./histogram.h"
#include "./memory.h"
#include "./port.h"
#include "./quality.h"
#if defined(__cplusplus) || defined(c_plusplus)
@@ -67,15 +67,18 @@ static BROTLI_INLINE void DestroyMetaBlockSplit(
BROTLI_FREE(m, mb->distance_histograms);
}
/* Uses the slow shortest-path block splitter and does context clustering. */
/* Uses the slow shortest-path block splitter and does context clustering.
The distance parameters are dynamically selected based on the commands
which get recomputed under the new distance parameters. The new distance
parameters are stored into *params. */
BROTLI_INTERNAL void BrotliBuildMetaBlock(MemoryManager* m,
const uint8_t* ringbuffer,
const size_t pos,
const size_t mask,
const BrotliEncoderParams* params,
BrotliEncoderParams* params,
uint8_t prev_byte,
uint8_t prev_byte2,
const Command* cmds,
Command* cmds,
size_t num_commands,
ContextType literal_context_mode,
MetaBlockSplit* mb);
@@ -85,14 +88,16 @@ BROTLI_INTERNAL void BrotliBuildMetaBlock(MemoryManager* m,
is the same for all block types. */
BROTLI_INTERNAL void BrotliBuildMetaBlockGreedy(
MemoryManager* m, const uint8_t* ringbuffer, size_t pos, size_t mask,
uint8_t prev_byte, uint8_t prev_byte2, ContextType literal_context_mode,
uint8_t prev_byte, uint8_t prev_byte2, ContextLut literal_context_lut,
size_t num_contexts, const uint32_t* static_context_map,
const Command* commands, size_t n_commands, MetaBlockSplit* mb);
BROTLI_INTERNAL void BrotliOptimizeHistograms(size_t num_direct_distance_codes,
size_t distance_postfix_bits,
BROTLI_INTERNAL void BrotliOptimizeHistograms(uint32_t num_distance_codes,
MetaBlockSplit* mb);
BROTLI_INTERNAL void BrotliInitDistanceParams(BrotliEncoderParams* params,
uint32_t npostfix, uint32_t ndirect);
#if defined(__cplusplus) || defined(c_plusplus)
} /* extern "C" */
#endif

2
C/brotli/enc/metablock_inc.h
View File

@@ -67,7 +67,7 @@ static void FN(InitBlockSplitter)(
split->lengths, split->lengths_alloc_size, max_num_blocks);
if (BROTLI_IS_OOM(m)) return;
self->split_->num_blocks = max_num_blocks;
assert(*histograms == 0);
BROTLI_DCHECK(*histograms == 0);
*histograms_size = max_num_types;
*histograms = BROTLI_ALLOC(m, HistogramType, *histograms_size);
self->histograms_ = *histograms;

44
C/brotli/enc/params.h Normal file
View File

@@ -0,0 +1,44 @@
/* Copyright 2017 Google Inc. All Rights Reserved.
Distributed under MIT license.
See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
*/
/* Parameters for the Brotli encoder with chosen quality levels. */
#ifndef BROTLI_ENC_PARAMS_H_
#define BROTLI_ENC_PARAMS_H_
#include "../encode.h"
#include "./encoder_dict.h"
typedef struct BrotliHasherParams {
int type;
int bucket_bits;
int block_bits;
int hash_len;
int num_last_distances_to_check;
} BrotliHasherParams;
typedef struct BrotliDistanceParams {
uint32_t distance_postfix_bits;
uint32_t num_direct_distance_codes;
uint32_t alphabet_size;
size_t max_distance;
} BrotliDistanceParams;
/* Encoding parameters */
typedef struct BrotliEncoderParams {
BrotliEncoderMode mode;
int quality;
int lgwin;
int lgblock;
size_t size_hint;
BROTLI_BOOL disable_literal_context_modeling;
BROTLI_BOOL large_window;
BrotliHasherParams hasher;
BrotliDistanceParams dist;
BrotliEncoderDictionary dictionary;
} BrotliEncoderParams;
#endif /* BROTLI_ENC_PARAMS_H_ */

184
C/brotli/enc/port.h
View File

@@ -1,184 +0,0 @@
/* Copyright 2013 Google Inc. All Rights Reserved.
Distributed under MIT license.
See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
*/
/* Macros for endianness, branch prediction and unaligned loads and stores. */
#ifndef BROTLI_ENC_PORT_H_
#define BROTLI_ENC_PORT_H_
#include <assert.h>
#include <string.h> /* memcpy */
#include "../port.h"
#include "../types.h"
#if defined OS_LINUX || defined OS_CYGWIN
#include <endian.h>
#elif defined OS_FREEBSD
#include <machine/endian.h>
#elif defined OS_MACOSX
#include <machine/endian.h>
/* Let's try and follow the Linux convention */
#define __BYTE_ORDER BYTE_ORDER
#define __LITTLE_ENDIAN LITTLE_ENDIAN
#endif
/* define the macro BROTLI_LITTLE_ENDIAN
using the above endian definitions from endian.h if
endian.h was included */
#ifdef __BYTE_ORDER
#if __BYTE_ORDER == __LITTLE_ENDIAN
#define BROTLI_LITTLE_ENDIAN
#endif
#else
#if defined(__LITTLE_ENDIAN__)
#define BROTLI_LITTLE_ENDIAN
#endif
#endif /* __BYTE_ORDER */
#if defined(__BYTE_ORDER__) && (__BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__)
#define BROTLI_LITTLE_ENDIAN
#endif
/* Enable little-endian optimization for x64 architecture on Windows. */
#if (defined(_WIN32) || defined(_WIN64)) && defined(_M_X64)
#define BROTLI_LITTLE_ENDIAN
#endif
/* Portable handling of unaligned loads, stores, and copies.
On some platforms, like ARM, the copy functions can be more efficient
then a load and a store. */
#if defined(BROTLI_LITTLE_ENDIAN) && (\
defined(ARCH_PIII) || defined(ARCH_ATHLON) || \
defined(ARCH_K8) || defined(_ARCH_PPC))
/* x86 and x86-64 can perform unaligned loads/stores directly;
modern PowerPC hardware can also do unaligned integer loads and stores;
but note: the FPU still sends unaligned loads and stores to a trap handler!
*/
#define BROTLI_UNALIGNED_LOAD32(_p) (*(const uint32_t *)(_p))
#define BROTLI_UNALIGNED_LOAD64LE(_p) (*(const uint64_t *)(_p))
#define BROTLI_UNALIGNED_STORE64LE(_p, _val) \
(*(uint64_t *)(_p) = (_val))
#elif defined(BROTLI_LITTLE_ENDIAN) && defined(__arm__) && \
!defined(__ARM_ARCH_5__) && \
!defined(__ARM_ARCH_5T__) && \
!defined(__ARM_ARCH_5TE__) && \
!defined(__ARM_ARCH_5TEJ__) && \
!defined(__ARM_ARCH_6__) && \
!defined(__ARM_ARCH_6J__) && \
!defined(__ARM_ARCH_6K__) && \
!defined(__ARM_ARCH_6Z__) && \
!defined(__ARM_ARCH_6ZK__) && \
!defined(__ARM_ARCH_6T2__)
/* ARMv7 and newer support native unaligned accesses, but only of 16-bit
and 32-bit values (not 64-bit); older versions either raise a fatal signal,
do an unaligned read and rotate the words around a bit, or do the reads very
slowly (trip through kernel mode). */
#define BROTLI_UNALIGNED_LOAD32(_p) (*(const uint32_t *)(_p))
static BROTLI_INLINE uint64_t BROTLI_UNALIGNED_LOAD64LE(const void *p) {
uint64_t t;
memcpy(&t, p, sizeof t);
return t;
}
static BROTLI_INLINE void BROTLI_UNALIGNED_STORE64LE(void *p, uint64_t v) {
memcpy(p, &v, sizeof v);
}
#else
/* These functions are provided for architectures that don't support */
/* unaligned loads and stores. */
static BROTLI_INLINE uint32_t BROTLI_UNALIGNED_LOAD32(const void *p) {
uint32_t t;
memcpy(&t, p, sizeof t);
return t;
}
#if defined(BROTLI_LITTLE_ENDIAN)
static BROTLI_INLINE uint64_t BROTLI_UNALIGNED_LOAD64LE(const void *p) {
uint64_t t;
memcpy(&t, p, sizeof t);
return t;
}
static BROTLI_INLINE void BROTLI_UNALIGNED_STORE64LE(void *p, uint64_t v) {
memcpy(p, &v, sizeof v);
}
#else /* BROTLI_LITTLE_ENDIAN */
static BROTLI_INLINE uint64_t BROTLI_UNALIGNED_LOAD64LE(const void *p) {
const uint8_t* in = (const uint8_t*)p;
uint64_t value = (uint64_t)(in[0]);
value |= (uint64_t)(in[1]) << 8;
value |= (uint64_t)(in[2]) << 16;
value |= (uint64_t)(in[3]) << 24;
value |= (uint64_t)(in[4]) << 32;
value |= (uint64_t)(in[5]) << 40;
value |= (uint64_t)(in[6]) << 48;
value |= (uint64_t)(in[7]) << 56;
return value;
}
static BROTLI_INLINE void BROTLI_UNALIGNED_STORE64LE(void *p, uint64_t v) {
uint8_t* out = (uint8_t*)p;
out[0] = (uint8_t)v;
out[1] = (uint8_t)(v >> 8);
out[2] = (uint8_t)(v >> 16);
out[3] = (uint8_t)(v >> 24);
out[4] = (uint8_t)(v >> 32);
out[5] = (uint8_t)(v >> 40);
out[6] = (uint8_t)(v >> 48);
out[7] = (uint8_t)(v >> 56);
}
#endif /* BROTLI_LITTLE_ENDIAN */
#endif
#define TEMPLATE_(T) \
static BROTLI_INLINE T brotli_min_ ## T (T a, T b) { return a < b ? a : b; } \
static BROTLI_INLINE T brotli_max_ ## T (T a, T b) { return a > b ? a : b; }
TEMPLATE_(double) TEMPLATE_(float) TEMPLATE_(int)
TEMPLATE_(size_t) TEMPLATE_(uint32_t) TEMPLATE_(uint8_t)
#undef TEMPLATE_
#define BROTLI_MIN(T, A, B) (brotli_min_ ## T((A), (B)))
#define BROTLI_MAX(T, A, B) (brotli_max_ ## T((A), (B)))
#define BROTLI_SWAP(T, A, I, J) { \
T __brotli_swap_tmp = (A)[(I)]; \
(A)[(I)] = (A)[(J)]; \
(A)[(J)] = __brotli_swap_tmp; \
}
#define BROTLI_ENSURE_CAPACITY(M, T, A, C, R) { \
if (C < (R)) { \
size_t _new_size = (C == 0) ? (R) : C; \
T* new_array; \
while (_new_size < (R)) _new_size *= 2; \
new_array = BROTLI_ALLOC((M), T, _new_size); \
if (!BROTLI_IS_OOM(m) && C != 0) \
memcpy(new_array, A, C * sizeof(T)); \
BROTLI_FREE((M), A); \
A = new_array; \
C = _new_size; \
} \
}
#endif /* BROTLI_ENC_PORT_H_ */

7
C/brotli/enc/prefix.h
View File

@@ -11,7 +11,7 @@
#define BROTLI_ENC_PREFIX_H_
#include "../common/constants.h"
#include "../port.h"
#include "../common/platform.h"
#include "../types.h"
#include "./fast_log.h"
@@ -39,11 +39,10 @@ static BROTLI_INLINE void PrefixEncodeCopyDistance(size_t distance_code,
size_t prefix = (dist >> bucket) & 1;
size_t offset = (2 + prefix) << bucket;
size_t nbits = bucket - postfix_bits;
*code = (uint16_t)(
*code = (uint16_t)((nbits << 10) |
(BROTLI_NUM_DISTANCE_SHORT_CODES + num_direct_codes +
((2 * (nbits - 1) + prefix) << postfix_bits) + postfix));
*extra_bits = (uint32_t)(
(nbits << 24) | ((dist - offset) >> postfix_bits));
*extra_bits = (uint32_t)((dist - offset) >> postfix_bits);
}
}

53
C/brotli/enc/quality.h
View File

@@ -10,7 +10,9 @@
#ifndef BROTLI_ENC_QUALITY_H_
#define BROTLI_ENC_QUALITY_H_
#include "../common/platform.h"
#include "../encode.h"
#include "./params.h"
#define FAST_ONE_PASS_COMPRESSION_QUALITY 0
#define FAST_TWO_PASS_COMPRESSION_QUALITY 1
@@ -19,36 +21,16 @@
#define MAX_QUALITY_FOR_STATIC_ENTROPY_CODES 2
#define MIN_QUALITY_FOR_BLOCK_SPLIT 4
#define MIN_QUALITY_FOR_NONZERO_DISTANCE_PARAMS 4
#define MIN_QUALITY_FOR_OPTIMIZE_HISTOGRAMS 4
#define MIN_QUALITY_FOR_EXTENSIVE_REFERENCE_SEARCH 5
#define MIN_QUALITY_FOR_CONTEXT_MODELING 5
#define MIN_QUALITY_FOR_HQ_CONTEXT_MODELING 7
#define MIN_QUALITY_FOR_HQ_BLOCK_SPLITTING 10
/* Only for "font" mode. */
#define MIN_QUALITY_FOR_RECOMPUTE_DISTANCE_PREFIXES 10
/* For quality below MIN_QUALITY_FOR_BLOCK_SPLIT there is no block splitting,
so we buffer at most this much literals and commands. */
#define MAX_NUM_DELAYED_SYMBOLS 0x2fff
typedef struct BrotliHasherParams {
int type;
int bucket_bits;
int block_bits;
int hash_len;
int num_last_distances_to_check;
} BrotliHasherParams;
/* Encoding parameters */
typedef struct BrotliEncoderParams {
BrotliEncoderMode mode;
int quality;
int lgwin;
int lgblock;
size_t size_hint;
BROTLI_BOOL disable_literal_context_modeling;
BrotliHasherParams hasher;
} BrotliEncoderParams;
#define MAX_NUM_DELAYED_SYMBOLS 0x2FFF
/* Returns hash-table size for quality levels 0 and 1. */
static BROTLI_INLINE size_t MaxHashTableSize(int quality) {
@@ -77,10 +59,15 @@ static BROTLI_INLINE size_t MaxZopfliCandidates(
static BROTLI_INLINE void SanitizeParams(BrotliEncoderParams* params) {
params->quality = BROTLI_MIN(int, BROTLI_MAX_QUALITY,
BROTLI_MAX(int, BROTLI_MIN_QUALITY, params->quality));
if (params->quality <= MAX_QUALITY_FOR_STATIC_ENTROPY_CODES) {
params->large_window = BROTLI_FALSE;
}
if (params->lgwin < BROTLI_MIN_WINDOW_BITS) {
params->lgwin = BROTLI_MIN_WINDOW_BITS;
} else if (params->lgwin > BROTLI_MAX_WINDOW_BITS) {
params->lgwin = BROTLI_MAX_WINDOW_BITS;
} else {
int max_lgwin = params->large_window ? BROTLI_LARGE_MAX_WINDOW_BITS :
BROTLI_MAX_WINDOW_BITS;
if (params->lgwin > max_lgwin) params->lgwin = max_lgwin;
}
}
@@ -155,6 +142,24 @@ static BROTLI_INLINE void ChooseHasher(const BrotliEncoderParams* params,
hparams->num_last_distances_to_check =
params->quality < 7 ? 4 : params->quality < 9 ? 10 : 16;
}
if (params->lgwin > 24) {
/* Different hashers for large window brotli: not for qualities <= 2,
these are too fast for large window. Not for qualities >= 10: their
hasher already works well with large window. So the changes are:
H3 --> H35: for quality 3.
H54 --> H55: for quality 4 with size hint > 1MB
H6 --> H65: for qualities 5, 6, 7, 8, 9. */
if (hparams->type == 3) {
hparams->type = 35;
}
if (hparams->type == 54) {
hparams->type = 55;
}
if (hparams->type == 6) {
hparams->type = 65;
}
}
}
#endif /* BROTLI_ENC_QUALITY_H_ */

26
C/brotli/enc/ringbuffer.h
View File

@@ -11,9 +11,9 @@
#include <string.h> /* memcpy */
#include "../common/platform.h"
#include "../types.h"
#include "./memory.h"
#include "./port.h"
#include "./quality.h"
#if defined(__cplusplus) || defined(c_plusplus)
@@ -41,9 +41,9 @@ typedef struct RingBuffer {
uint32_t pos_;
/* The actual ring buffer containing the copy of the last two bytes, the data,
and the copy of the beginning as a tail. */
uint8_t *data_;
uint8_t* data_;
/* The start of the ring-buffer. */
uint8_t *buffer_;
uint8_t* buffer_;
} RingBuffer;
static BROTLI_INLINE void RingBufferInit(RingBuffer* rb) {
@@ -91,7 +91,7 @@ static BROTLI_INLINE void RingBufferInitBuffer(
}
static BROTLI_INLINE void RingBufferWriteTail(
const uint8_t *bytes, size_t n, RingBuffer* rb) {
const uint8_t* bytes, size_t n, RingBuffer* rb) {
const size_t masked_pos = rb->pos_ & rb->mask_;
if (BROTLI_PREDICT_FALSE(masked_pos < rb->tail_size_)) {
/* Just fill the tail buffer with the beginning data. */
@@ -103,7 +103,7 @@ static BROTLI_INLINE void RingBufferWriteTail(
/* Push bytes into the ring buffer. */
static BROTLI_INLINE void RingBufferWrite(
MemoryManager* m, const uint8_t *bytes, size_t n, RingBuffer* rb) {
MemoryManager* m, const uint8_t* bytes, size_t n, RingBuffer* rb) {
if (rb->pos_ == 0 && n < rb->tail_size_) {
/* Special case for the first write: to process the first block, we don't
need to allocate the whole ring-buffer and we don't need the tail
@@ -144,12 +144,16 @@ static BROTLI_INLINE void RingBufferWrite(
n - (rb->size_ - masked_pos));
}
}
rb->buffer_[-2] = rb->buffer_[rb->size_ - 2];
rb->buffer_[-1] = rb->buffer_[rb->size_ - 1];
rb->pos_ += (uint32_t)n;
if (rb->pos_ > (1u << 30)) {
/* Wrap, but preserve not-a-first-lap feature. */
rb->pos_ = (rb->pos_ & ((1u << 30) - 1)) | (1u << 30);
{
BROTLI_BOOL not_first_lap = (rb->pos_ & (1u << 31)) != 0;
uint32_t rb_pos_mask = (1u << 31) - 1;
rb->buffer_[-2] = rb->buffer_[rb->size_ - 2];
rb->buffer_[-1] = rb->buffer_[rb->size_ - 1];
rb->pos_ = (rb->pos_ & rb_pos_mask) + (uint32_t)(n & rb_pos_mask);
if (not_first_lap) {
/* Wrap, but preserve not-a-first-lap feature. */
rb->pos_ |= 1u << 31;
}
}
}

7
C/brotli/enc/static_dict.h
View File

@@ -10,15 +10,16 @@
#define BROTLI_ENC_STATIC_DICT_H_
#include "../common/dictionary.h"
#include "../common/platform.h"
#include "../types.h"
#include "./port.h"
#include "./encoder_dict.h"
#if defined(__cplusplus) || defined(c_plusplus)
extern "C" {
#endif
#define BROTLI_MAX_STATIC_DICTIONARY_MATCH_LEN 37
static const uint32_t kInvalidMatch = 0xfffffff;
static const uint32_t kInvalidMatch = 0xFFFFFFF;
/* Matches data against static dictionary words, and for each length l,
for which a match is found, updates matches[l] to be the minimum possible
@@ -28,7 +29,7 @@ static const uint32_t kInvalidMatch = 0xfffffff;
matches array is at least BROTLI_MAX_STATIC_DICTIONARY_MATCH_LEN + 1 long
all elements are initialized to kInvalidMatch */
BROTLI_INTERNAL BROTLI_BOOL BrotliFindAllStaticDictionaryMatches(
const BrotliDictionary* dictionary,
const BrotliEncoderDictionary* dictionary,
const uint8_t* data, size_t min_length, size_t max_length,
uint32_t* matches);

2
C/brotli/enc/static_dict_lut.h
View File

@@ -23,7 +23,7 @@ typedef struct DictWord {
} DictWord;
static const int kDictNumBits = 15;
static const uint32_t kDictHashMul32 = 0x1e35a7bd;
static const uint32_t kDictHashMul32 = 0x1E35A7BD;
static const uint16_t kStaticDictionaryBuckets[32768] = {
1,0,0,0,0,0,0,0,0,3,6,0,0,0,0,0,20,0,0,0,21,0,22,0,0,0,0,0,0,0,0,23,0,0,25,0,29,

2
C/brotli/enc/utf8_util.h
View File

@@ -9,8 +9,8 @@
#ifndef BROTLI_ENC_UTF8_UTIL_H_
#define BROTLI_ENC_UTF8_UTIL_H_
#include "../common/platform.h"
#include "../types.h"
#include "./port.h"
#if defined(__cplusplus) || defined(c_plusplus)
extern "C" {

37
C/brotli/enc/write_bits.h
View File

@@ -9,11 +9,8 @@
#ifndef BROTLI_ENC_WRITE_BITS_H_
#define BROTLI_ENC_WRITE_BITS_H_
#include <assert.h>
#include <stdio.h> /* printf */
#include "../common/platform.h"
#include "../types.h"
#include "./port.h"
#if defined(__cplusplus) || defined(c_plusplus)
extern "C" {
@@ -38,27 +35,27 @@ extern "C" {
and locate the rest in BYTE+1, BYTE+2, etc. */
static BROTLI_INLINE void BrotliWriteBits(size_t n_bits,
uint64_t bits,
size_t * BROTLI_RESTRICT pos,
uint8_t * BROTLI_RESTRICT array) {
#ifdef BROTLI_LITTLE_ENDIAN
size_t* BROTLI_RESTRICT pos,
uint8_t* BROTLI_RESTRICT array) {
#if defined(BROTLI_LITTLE_ENDIAN)
/* This branch of the code can write up to 56 bits at a time,
7 bits are lost by being perhaps already in *p and at least
1 bit is needed to initialize the bit-stream ahead (i.e. if 7
bits are in *p and we write 57 bits, then the next write will
access a byte that was never initialized). */
uint8_t *p = &array[*pos >> 3];
uint64_t v = *p;
#ifdef BIT_WRITER_DEBUG
printf("WriteBits %2d 0x%016llx %10d\n", n_bits, bits, *pos);
#endif
assert((bits >> n_bits) == 0);
assert(n_bits <= 56);
uint8_t* p = &array[*pos >> 3];
uint64_t v = (uint64_t)(*p); /* Zero-extend 8 to 64 bits. */
BROTLI_LOG(("WriteBits %2d 0x%08x%08x %10d\n", (int)n_bits,
(uint32_t)(bits >> 32), (uint32_t)(bits & 0xFFFFFFFF),
(int)*pos));
BROTLI_DCHECK((bits >> n_bits) == 0);
BROTLI_DCHECK(n_bits <= 56);
v |= bits << (*pos & 7);
BROTLI_UNALIGNED_STORE64LE(p, v); /* Set some bits. */
*pos += n_bits;
#else
/* implicit & 0xff is assumed for uint8_t arithmetics */
uint8_t *array_pos = &array[*pos >> 3];
/* implicit & 0xFF is assumed for uint8_t arithmetics */
uint8_t* array_pos = &array[*pos >> 3];
const size_t bits_reserved_in_first_byte = (*pos & 7);
size_t bits_left_to_write;
bits <<= bits_reserved_in_first_byte;
@@ -75,11 +72,9 @@ static BROTLI_INLINE void BrotliWriteBits(size_t n_bits,
}
static BROTLI_INLINE void BrotliWriteBitsPrepareStorage(
size_t pos, uint8_t *array) {
#ifdef BIT_WRITER_DEBUG
printf("WriteBitsPrepareStorage %10d\n", pos);
#endif
assert((pos & 7) == 0);
size_t pos, uint8_t* array) {
BROTLI_LOG(("WriteBitsPrepareStorage %10d\n", (int)pos));
BROTLI_DCHECK((pos & 7) == 0);
array[pos >> 3] = 0;
}