xevion/easy7zip
1
0
Fork 0
mirror of https://github.com/Xevion/easy7zip.git synced 2025-12-10 14:07:11 -06:00
Code Issues Packages Projects Releases Wiki Activity

Add options to brotli and implement clean brotli .br support

Browse Source 
- allow to specify brotli window size
  - parameter -m0=brotli:long=n, BROTLI_MAX_WINDOW_BITS (24) used by default in brotli-mt, smaller == faster
  - note that :long can be set up to BROTLI_LARGE_MAX_WINDOW_BITS (30), whereas :wlog can be set up to BROTLI_MAX_WINDOW_BITS (24) only...
  - todo: check whether set of BROTLI_PARAM_LARGE_WINDOW to BROTLI_TRUE is needed if (lgwin > BROTLI_MAX_WINDOW_BITS)

- implementation of single-threaded brotli compression / decompression for .br data

Signed-off-by: Sergey G. Brester <info@sebres.de>
Reviewed-by: Tino Reichardt <milky-7zip@mcmilk.de>
This commit is contained in:
sebres
2021-08-30 17:16:08 +02:00
committed by Tino Reichardt
parent 9bb11a56b6
commit eeae03eaa1
8 changed files with 660 additions and 40 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
C/zstdmt/brotli-mt.h
View File

@@ -101,7 +101,7 @@ typedef struct BROTLIMT_CCtx_s BROTLIMT_CCtx;
* @inputsize - if zero, becomes some optimal value for the level
* - if nonzero, the given value is taken
*/
BROTLIMT_CCtx *BROTLIMT_createCCtx(int threads, int level, int inputsize);
BROTLIMT_CCtx *BROTLIMT_createCCtx(int threads, uint64_t unpackSize, int level, int inputsize, int lgwin);
/**
* 2) threaded compression

174
C/zstdmt/brotli-mt_compress.c
View File

@@ -15,6 +15,7 @@
#include <string.h>
#include "encode.h"
#include "common/constants.h"
#include "brotli-mt.h"
#include "memmt.h"
@@ -52,12 +53,17 @@ struct BROTLIMT_CCtx_s {
/* levels: 1..BROTLIMT_LEVEL_MAX */
int level;
/* threads: 1..BROTLIMT_THREAD_MAX */
/* threads: 0..BROTLIMT_THREAD_MAX */
int threads;
/* size of file/stream to compress if known */
uint64_t unpackSize;
/* should be used for read from input */
int inputsize;
int lgwin;
/* statistic */
size_t insize;
size_t outsize;
@@ -87,7 +93,7 @@ struct BROTLIMT_CCtx_s {
* Compression
****************************************/
BROTLIMT_CCtx *BROTLIMT_createCCtx(int threads, int level, int inputsize)
BROTLIMT_CCtx *BROTLIMT_createCCtx(int threads, uint64_t unpackSize, int level, int inputsize, int lgwin)
{
BROTLIMT_CCtx *ctx;
int t;
@@ -98,7 +104,7 @@ BROTLIMT_CCtx *BROTLIMT_createCCtx(int threads, int level, int inputsize)
return 0;
/* check threads value */
if (threads < 1 || threads > BROTLIMT_THREAD_MAX)
if (threads < 0 || threads > BROTLIMT_THREAD_MAX)
return 0;
/* check level */
@@ -111,30 +117,37 @@ BROTLIMT_CCtx *BROTLIMT_createCCtx(int threads, int level, int inputsize)
else
ctx->inputsize = 1024 * 1024 * (level ? level : 1);
ctx->lgwin = lgwin;
/* setup ctx */
ctx->level = level;
ctx->threads = threads;
ctx->unpackSize = unpackSize;
ctx->insize = 0;
ctx->outsize = 0;
ctx->frames = 0;
ctx->curframe = 0;
pthread_mutex_init(&ctx->read_mutex, NULL);
pthread_mutex_init(&ctx->write_mutex, NULL);
if (threads) {
pthread_mutex_init(&ctx->read_mutex, NULL);
pthread_mutex_init(&ctx->write_mutex, NULL);
/* free -> busy -> out -> free -> ... */
INIT_LIST_HEAD(&ctx->writelist_free); /* free, can be used */
INIT_LIST_HEAD(&ctx->writelist_busy); /* busy */
INIT_LIST_HEAD(&ctx->writelist_done); /* can be written */
/* free -> busy -> out -> free -> ... */
INIT_LIST_HEAD(&ctx->writelist_free); /* free, can be used */
INIT_LIST_HEAD(&ctx->writelist_busy); /* busy */
INIT_LIST_HEAD(&ctx->writelist_done); /* can be written */
ctx->cwork = (cwork_t *) malloc(sizeof(cwork_t) * threads);
if (!ctx->cwork)
goto err_cwork;
ctx->cwork = (cwork_t *) malloc(sizeof(cwork_t) * threads);
if (!ctx->cwork)
goto err_cwork;
for (t = 0; t < threads; t++) {
cwork_t *w = &ctx->cwork[t];
w->ctx = ctx;
}
for (t = 0; t < threads; t++) {
cwork_t *w = &ctx->cwork[t];
w->ctx = ctx;
}
} else {
ctx->cwork = NULL;
}
return ctx;
@@ -269,7 +282,7 @@ static void *pt_compress(void *arg)
uint8_t *obuf = (uint8_t*)wl->out.buf + 16;
wl->out.size -= 16;
rv = BrotliEncoderCompress(ctx->level,
BROTLI_MAX_WINDOW_BITS,
ctx->lgwin,
BROTLI_MODE_GENERIC, in.size,
ibuf, &wl->out.size, obuf);
@@ -319,6 +332,119 @@ static void *pt_compress(void *arg)
return 0;
}
/* single threaded (standard brotli stream, without header/mt-frames) */
static size_t st_compress(void *arg)
{
BROTLIMT_CCtx *ctx = (BROTLIMT_CCtx *) arg;
BrotliEncoderOperation brop = BROTLI_OPERATION_PROCESS;
BROTLIMT_Buffer Out;
BROTLIMT_Buffer *out = &Out;
BROTLIMT_Buffer In;
BROTLIMT_Buffer *in = &In;
BrotliEncoderState *state;
const uint8_t* next_in;
uint8_t* next_out;
int rv;
size_t retval = 0;
/* allocate space for input buffer (default 1M * level) */
in->allocated = ctx->inputsize;
in->buf = malloc(in->allocated);
if (!in->buf)
return MT_ERROR(memory_allocation);
next_in = in->buf;
in->size = 0;
/* allocate space for output buffer */
out->allocated = out->size = ctx->inputsize / 4;
out->buf = malloc(out->size);
if (!out->buf) {
free(in->buf);
return MT_ERROR(memory_allocation);
}
next_out = out->buf;
state = BrotliEncoderCreateInstance(NULL, NULL, NULL);
if (!state) {
free(in->buf);
free(out->buf);
return MT_ERROR(memory_allocation);
}
BrotliEncoderSetParameter(state, BROTLI_PARAM_QUALITY, (uint32_t)ctx->level);
if (ctx->lgwin > 0) {
/* Specified by user. */
/* Do not enable "large-window" extension, if not required. */
if (ctx->lgwin > BROTLI_MAX_WINDOW_BITS) {
BrotliEncoderSetParameter(state, BROTLI_PARAM_LARGE_WINDOW, 1u);
}
BrotliEncoderSetParameter(state, BROTLI_PARAM_LGWIN, (uint32_t)ctx->lgwin);
} else {
/* 0, or not specified by user; could be chosen by compressor. */
uint32_t lgwin = 24 /* DEFAULT_LGWIN */;
/* Use file size to limit lgwin. */
if (ctx->unpackSize >= 0) {
lgwin = BROTLI_MIN_WINDOW_BITS;
while (BROTLI_MAX_BACKWARD_LIMIT(lgwin) <
(uint64_t)ctx->unpackSize) {
lgwin++;
if (lgwin == BROTLI_MAX_WINDOW_BITS) break;
}
}
BrotliEncoderSetParameter(state, BROTLI_PARAM_LGWIN, lgwin);
}
if (ctx->unpackSize > 0) {
uint32_t size_hint = ctx->unpackSize < (1 << 30) ?
(uint32_t)ctx->unpackSize : (1u << 30);
BrotliEncoderSetParameter(state, BROTLI_PARAM_SIZE_HINT, size_hint);
}
while (1) {
if (in->size == 0 && brop != BROTLI_OPERATION_FINISH) {
in->size = in->allocated;
rv = ctx->fn_read(ctx->arg_read, in);
if (rv != 0) {
retval = mt_error(rv);
goto done;
}
if (in->size == 0) {
brop = BROTLI_OPERATION_FINISH; // eof
}
next_in = in->buf;
}
if (!BrotliEncoderCompressStream(state, brop, &in->size, &next_in, &out->size, &next_out, 0)) {
retval = MT_ERROR(frame_compress);
goto done;
}
if (out->size == 0) {
out->size = next_out - (uint8_t*)out->buf;
rv = ctx->fn_write(ctx->arg_write, out);
if (rv != 0) {
retval = mt_error(rv);
goto done;
}
next_out = out->buf;
out->size = out->allocated;
}
if (BrotliEncoderIsFinished(state)) {
out->size = next_out - (uint8_t*)out->buf;
rv = ctx->fn_write(ctx->arg_write, out);
if (rv != 0) {
retval = mt_error(rv);
goto done;
}
break;
}
}
done:
free(in->buf);
free(out->buf);
BrotliEncoderDestroyInstance(state);
return retval;
}
size_t BROTLIMT_compressCCtx(BROTLIMT_CCtx * ctx, BROTLIMT_RdWr_t * rdwr)
{
int t;
@@ -333,6 +459,12 @@ size_t BROTLIMT_compressCCtx(BROTLIMT_CCtx * ctx, BROTLIMT_RdWr_t * rdwr)
ctx->arg_read = rdwr->arg_read;
ctx->arg_write = rdwr->arg_write;
/* single threaded brotli (no header, no mt-frames) */
if (ctx->threads == 0) {
/* decompress single threaded */
return st_compress(ctx);
}
/* start all workers */
for (t = 0; t < ctx->threads; t++) {
cwork_t *w = &ctx->cwork[t];
@@ -394,9 +526,11 @@ void BROTLIMT_freeCCtx(BROTLIMT_CCtx * ctx)
if (!ctx)
return;
pthread_mutex_destroy(&ctx->read_mutex);
pthread_mutex_destroy(&ctx->write_mutex);
free(ctx->cwork);
if (ctx->threads) {
pthread_mutex_destroy(&ctx->read_mutex);
pthread_mutex_destroy(&ctx->write_mutex);
free(ctx->cwork);
}
free(ctx);
ctx = 0;

120
C/zstdmt/brotli-mt_decompress.c
View File

@@ -50,7 +50,7 @@ struct writelist {
struct BROTLIMT_DCtx_s {
/* threads: 1..BROTLIMT_THREAD_MAX */
/* threads: 0, 1..BROTLIMT_THREAD_MAX */
int threads;
/* should be used for read from input */
@@ -96,7 +96,7 @@ BROTLIMT_DCtx *BROTLIMT_createDCtx(int threads, int inputsize)
return 0;
/* check threads value */
if (threads < 1 || threads > BROTLIMT_THREAD_MAX)
if (threads < 0 || threads > BROTLIMT_THREAD_MAX)
return 0;
/* setup ctx */
@@ -112,12 +112,16 @@ BROTLIMT_DCtx *BROTLIMT_createDCtx(int threads, int inputsize)
else
ctx->inputsize = 1024 * 64; /* 64K buffer */
pthread_mutex_init(&ctx->read_mutex, NULL);
pthread_mutex_init(&ctx->write_mutex, NULL);
if (threads) {
pthread_mutex_init(&ctx->read_mutex, NULL);
pthread_mutex_init(&ctx->write_mutex, NULL);
INIT_LIST_HEAD(&ctx->writelist_free);
INIT_LIST_HEAD(&ctx->writelist_busy);
INIT_LIST_HEAD(&ctx->writelist_done);
INIT_LIST_HEAD(&ctx->writelist_free);
INIT_LIST_HEAD(&ctx->writelist_busy);
INIT_LIST_HEAD(&ctx->writelist_done);
} else {
threads = 1;
}
ctx->cwork = (cwork_t *) malloc(sizeof(cwork_t) * threads);
if (!ctx->cwork)
@@ -376,6 +380,90 @@ static void *pt_decompress(void *arg)
return (void *)result;
}
/* single threaded (standard brotli stream, without header/mt-frames) */
static size_t st_decompress(void *arg)
{
BROTLIMT_DCtx *ctx = (BROTLIMT_DCtx *) arg;
BrotliDecoderResult bres = BROTLI_DECODER_RESULT_NEEDS_MORE_INPUT;
cwork_t *w = &ctx->cwork[0];
BROTLIMT_Buffer Out;
BROTLIMT_Buffer *out = &Out;
BROTLIMT_Buffer *in = &w->in;
BrotliDecoderState *state;
const uint8_t* next_in;
uint8_t* next_out;
int rv;
size_t retval = 0;
/* allocate space for input buffer */
in->allocated = in->size = ctx->inputsize;
in->buf = malloc(in->size);
if (!in->buf)
return MT_ERROR(memory_allocation);
next_in = in->buf;
/* allocate space for output buffer */
out->allocated = out->size = ctx->inputsize * 4;
out->buf = malloc(out->size);
if (!out->buf) {
free(in->buf);
return MT_ERROR(memory_allocation);
}
next_out = out->buf;
state = BrotliDecoderCreateInstance(NULL, NULL, NULL);
if (!state) {
free(in->buf);
free(out->buf);
return MT_ERROR(memory_allocation);
}
BrotliDecoderSetParameter(state, BROTLI_DECODER_PARAM_LARGE_WINDOW, 1);
while (1) {
if (bres == BROTLI_DECODER_RESULT_NEEDS_MORE_INPUT) {
in->size = in->allocated;
rv = ctx->fn_read(ctx->arg_read, in);
if (in->size == 0) break;
if (rv != 0) {
retval = mt_error(rv);
goto done;
}
next_in = in->buf;
} else if (bres == BROTLI_DECODER_RESULT_NEEDS_MORE_OUTPUT) {
out->size = next_out - (uint8_t*)out->buf;
rv = ctx->fn_write(ctx->arg_write, out);
if (rv != 0) {
retval = mt_error(rv);
goto done;
}
next_out = out->buf;
} else {
break;
}
bres = BrotliDecoderDecompressStream(state, &in->size, &next_in, &out->size, &next_out, 0);
}
if (bres != BROTLI_DECODER_RESULT_SUCCESS) {
retval = MT_ERROR(data_error); // corrupt input
goto done;
}
out->size = next_out - (uint8_t*)out->buf;
if (out->size != 0) {
rv = ctx->fn_write(ctx->arg_write, out);
if (rv != 0) {
retval = mt_error(rv);
goto done;
}
}
done:
free(in->buf);
free(out->buf);
BrotliDecoderDestroyInstance(state);
return retval;
}
size_t BROTLIMT_decompressDCtx(BROTLIMT_DCtx * ctx, BROTLIMT_RdWr_t * rdwr)
{
unsigned char buf[4];
@@ -393,6 +481,14 @@ size_t BROTLIMT_decompressDCtx(BROTLIMT_DCtx * ctx, BROTLIMT_RdWr_t * rdwr)
ctx->arg_read = rdwr->arg_read;
ctx->arg_write = rdwr->arg_write;
/* For single threaded brotli (no header, no mt-frames), don't check for
BROTLIMT_MAGIC_SKIPPABLE here, because stdandard brotli stream may also
start with it, so will be mistakenly considered as brotli-mt stream. */
if (ctx->threads == 0) {
/* decompress single threaded */
return st_decompress(ctx);
}
/* check for BROTLIMT_MAGIC_SKIPPABLE */
in->buf = buf;
in->size = 4;
@@ -402,10 +498,6 @@ size_t BROTLIMT_decompressDCtx(BROTLIMT_DCtx * ctx, BROTLIMT_RdWr_t * rdwr)
if (in->size != 4)
return MT_ERROR(data_error);
/* single threaded with unknown sizes */
if (MEM_readLE32(buf) != BROTLIMT_MAGIC_SKIPPABLE)
return MT_ERROR(data_error);
/* mark unused */
in->buf = 0;
in->size = 0;
@@ -485,8 +577,10 @@ void BROTLIMT_freeDCtx(BROTLIMT_DCtx * ctx)
if (!ctx)
return;
pthread_mutex_destroy(&ctx->read_mutex);
pthread_mutex_destroy(&ctx->write_mutex);
if (ctx->threads) {
pthread_mutex_destroy(&ctx->read_mutex);
pthread_mutex_destroy(&ctx->write_mutex);
}
free(ctx->cwork);
free(ctx);
ctx = 0;