xevion/easy7zip
1
0
Fork 0
mirror of https://github.com/Xevion/easy7zip.git synced 2025-12-10 22:07:08 -06:00
Code Issues Packages Projects Releases Wiki Activity

Add options to brotli and implement clean brotli .br support

Browse Source 
- allow to specify brotli window size
  - parameter -m0=brotli:long=n, BROTLI_MAX_WINDOW_BITS (24) used by default in brotli-mt, smaller == faster
  - note that :long can be set up to BROTLI_LARGE_MAX_WINDOW_BITS (30), whereas :wlog can be set up to BROTLI_MAX_WINDOW_BITS (24) only...
  - todo: check whether set of BROTLI_PARAM_LARGE_WINDOW to BROTLI_TRUE is needed if (lgwin > BROTLI_MAX_WINDOW_BITS)

- implementation of single-threaded brotli compression / decompression for .br data

Signed-off-by: Sergey G. Brester <info@sebres.de>
Reviewed-by: Tino Reichardt <milky-7zip@mcmilk.de>
This commit is contained in:
sebres
2021-08-30 17:16:08 +02:00
committed by Tino Reichardt
parent 9bb11a56b6
commit eeae03eaa1
8 changed files with 660 additions and 40 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
C/zstdmt/brotli-mt.h
View File

@@ -101,7 +101,7 @@ typedef struct BROTLIMT_CCtx_s BROTLIMT_CCtx;
* @inputsize - if zero, becomes some optimal value for the level * @inputsize - if zero, becomes some optimal value for the level
* - if nonzero, the given value is taken * - if nonzero, the given value is taken
*/ */
BROTLIMT_CCtx *BROTLIMT_createCCtx(int threads, int level, int inputsize); BROTLIMT_CCtx *BROTLIMT_createCCtx(int threads, uint64_t unpackSize, int level, int inputsize, int lgwin);
/** /**
* 2) threaded compression * 2) threaded compression

172
C/zstdmt/brotli-mt_compress.c
View File

@@ -15,6 +15,7 @@
#include <string.h> #include <string.h>
#include "encode.h" #include "encode.h"
#include "common/constants.h"
#include "brotli-mt.h" #include "brotli-mt.h"
#include "memmt.h" #include "memmt.h"
@@ -52,12 +53,17 @@ struct BROTLIMT_CCtx_s {
/* levels: 1..BROTLIMT_LEVEL_MAX */ /* levels: 1..BROTLIMT_LEVEL_MAX */
int level; int level;
/* threads: 1..BROTLIMT_THREAD_MAX */ /* threads: 0..BROTLIMT_THREAD_MAX */
int threads; int threads;
/* size of file/stream to compress if known */
uint64_t unpackSize;
/* should be used for read from input */ /* should be used for read from input */
int inputsize; int inputsize;
int lgwin;
/* statistic */ /* statistic */
size_t insize; size_t insize;
size_t outsize; size_t outsize;
@@ -87,7 +93,7 @@ struct BROTLIMT_CCtx_s {
* Compression * Compression
****************************************/ ****************************************/
BROTLIMT_CCtx *BROTLIMT_createCCtx(int threads, int level, int inputsize) BROTLIMT_CCtx *BROTLIMT_createCCtx(int threads, uint64_t unpackSize, int level, int inputsize, int lgwin)
{ {
BROTLIMT_CCtx *ctx; BROTLIMT_CCtx *ctx;
int t; int t;
@@ -98,7 +104,7 @@ BROTLIMT_CCtx *BROTLIMT_createCCtx(int threads, int level, int inputsize)
return 0; return 0;
/* check threads value */ /* check threads value */
if (threads < 1 || threads > BROTLIMT_THREAD_MAX) if (threads < 0 || threads > BROTLIMT_THREAD_MAX)
return 0; return 0;
/* check level */ /* check level */
@@ -111,29 +117,36 @@ BROTLIMT_CCtx *BROTLIMT_createCCtx(int threads, int level, int inputsize)
else else
ctx->inputsize = 1024 * 1024 * (level ? level : 1); ctx->inputsize = 1024 * 1024 * (level ? level : 1);
ctx->lgwin = lgwin;
/* setup ctx */ /* setup ctx */
ctx->level = level; ctx->level = level;
ctx->threads = threads; ctx->threads = threads;
ctx->unpackSize = unpackSize;
ctx->insize = 0; ctx->insize = 0;
ctx->outsize = 0; ctx->outsize = 0;
ctx->frames = 0; ctx->frames = 0;
ctx->curframe = 0; ctx->curframe = 0;
pthread_mutex_init(&ctx->read_mutex, NULL); if (threads) {
pthread_mutex_init(&ctx->write_mutex, NULL); pthread_mutex_init(&ctx->read_mutex, NULL);
pthread_mutex_init(&ctx->write_mutex, NULL);
/* free -> busy -> out -> free -> ... */ /* free -> busy -> out -> free -> ... */
INIT_LIST_HEAD(&ctx->writelist_free); /* free, can be used */ INIT_LIST_HEAD(&ctx->writelist_free); /* free, can be used */
INIT_LIST_HEAD(&ctx->writelist_busy); /* busy */ INIT_LIST_HEAD(&ctx->writelist_busy); /* busy */
INIT_LIST_HEAD(&ctx->writelist_done); /* can be written */ INIT_LIST_HEAD(&ctx->writelist_done); /* can be written */
ctx->cwork = (cwork_t *) malloc(sizeof(cwork_t) * threads); ctx->cwork = (cwork_t *) malloc(sizeof(cwork_t) * threads);
if (!ctx->cwork) if (!ctx->cwork)
goto err_cwork; goto err_cwork;
for (t = 0; t < threads; t++) { for (t = 0; t < threads; t++) {
cwork_t *w = &ctx->cwork[t]; cwork_t *w = &ctx->cwork[t];
w->ctx = ctx; w->ctx = ctx;
}
} else {
ctx->cwork = NULL;
} }
return ctx; return ctx;
@@ -269,7 +282,7 @@ static void *pt_compress(void *arg)
uint8_t *obuf = (uint8_t*)wl->out.buf + 16; uint8_t *obuf = (uint8_t*)wl->out.buf + 16;
wl->out.size -= 16; wl->out.size -= 16;
rv = BrotliEncoderCompress(ctx->level, rv = BrotliEncoderCompress(ctx->level,
BROTLI_MAX_WINDOW_BITS, ctx->lgwin,
BROTLI_MODE_GENERIC, in.size, BROTLI_MODE_GENERIC, in.size,
ibuf, &wl->out.size, obuf); ibuf, &wl->out.size, obuf);
@@ -319,6 +332,119 @@ static void *pt_compress(void *arg)
return 0; return 0;
} }
/* single threaded (standard brotli stream, without header/mt-frames) */
static size_t st_compress(void *arg)
{
BROTLIMT_CCtx *ctx = (BROTLIMT_CCtx *) arg;
BrotliEncoderOperation brop = BROTLI_OPERATION_PROCESS;
BROTLIMT_Buffer Out;
BROTLIMT_Buffer *out = &Out;
BROTLIMT_Buffer In;
BROTLIMT_Buffer *in = &In;
BrotliEncoderState *state;
const uint8_t* next_in;
uint8_t* next_out;
int rv;
size_t retval = 0;
/* allocate space for input buffer (default 1M * level) */
in->allocated = ctx->inputsize;
in->buf = malloc(in->allocated);
if (!in->buf)
return MT_ERROR(memory_allocation);
next_in = in->buf;
in->size = 0;
/* allocate space for output buffer */
out->allocated = out->size = ctx->inputsize / 4;
out->buf = malloc(out->size);
if (!out->buf) {
free(in->buf);
return MT_ERROR(memory_allocation);
}
next_out = out->buf;
state = BrotliEncoderCreateInstance(NULL, NULL, NULL);
if (!state) {
free(in->buf);
free(out->buf);
return MT_ERROR(memory_allocation);
}
BrotliEncoderSetParameter(state, BROTLI_PARAM_QUALITY, (uint32_t)ctx->level);
if (ctx->lgwin > 0) {
/* Specified by user. */
/* Do not enable "large-window" extension, if not required. */
if (ctx->lgwin > BROTLI_MAX_WINDOW_BITS) {
BrotliEncoderSetParameter(state, BROTLI_PARAM_LARGE_WINDOW, 1u);
}
BrotliEncoderSetParameter(state, BROTLI_PARAM_LGWIN, (uint32_t)ctx->lgwin);
} else {
/* 0, or not specified by user; could be chosen by compressor. */
uint32_t lgwin = 24 /* DEFAULT_LGWIN */;
/* Use file size to limit lgwin. */
if (ctx->unpackSize >= 0) {
lgwin = BROTLI_MIN_WINDOW_BITS;
while (BROTLI_MAX_BACKWARD_LIMIT(lgwin) <
(uint64_t)ctx->unpackSize) {
lgwin++;
if (lgwin == BROTLI_MAX_WINDOW_BITS) break;
}
}
BrotliEncoderSetParameter(state, BROTLI_PARAM_LGWIN, lgwin);
}
if (ctx->unpackSize > 0) {
uint32_t size_hint = ctx->unpackSize < (1 << 30) ?
(uint32_t)ctx->unpackSize : (1u << 30);
BrotliEncoderSetParameter(state, BROTLI_PARAM_SIZE_HINT, size_hint);
}
while (1) {
if (in->size == 0 && brop != BROTLI_OPERATION_FINISH) {
in->size = in->allocated;
rv = ctx->fn_read(ctx->arg_read, in);
if (rv != 0) {
retval = mt_error(rv);
goto done;
}
if (in->size == 0) {
brop = BROTLI_OPERATION_FINISH; // eof
}
next_in = in->buf;
}
if (!BrotliEncoderCompressStream(state, brop, &in->size, &next_in, &out->size, &next_out, 0)) {
retval = MT_ERROR(frame_compress);
goto done;
}
if (out->size == 0) {
out->size = next_out - (uint8_t*)out->buf;
rv = ctx->fn_write(ctx->arg_write, out);
if (rv != 0) {
retval = mt_error(rv);
goto done;
}
next_out = out->buf;
out->size = out->allocated;
}
if (BrotliEncoderIsFinished(state)) {
out->size = next_out - (uint8_t*)out->buf;
rv = ctx->fn_write(ctx->arg_write, out);
if (rv != 0) {
retval = mt_error(rv);
goto done;
}
break;
}
}
done:
free(in->buf);
free(out->buf);
BrotliEncoderDestroyInstance(state);
return retval;
}
size_t BROTLIMT_compressCCtx(BROTLIMT_CCtx * ctx, BROTLIMT_RdWr_t * rdwr) size_t BROTLIMT_compressCCtx(BROTLIMT_CCtx * ctx, BROTLIMT_RdWr_t * rdwr)
{ {
int t; int t;
@@ -333,6 +459,12 @@ size_t BROTLIMT_compressCCtx(BROTLIMT_CCtx * ctx, BROTLIMT_RdWr_t * rdwr)
ctx->arg_read = rdwr->arg_read; ctx->arg_read = rdwr->arg_read;
ctx->arg_write = rdwr->arg_write; ctx->arg_write = rdwr->arg_write;
/* single threaded brotli (no header, no mt-frames) */
if (ctx->threads == 0) {
/* decompress single threaded */
return st_compress(ctx);
}
/* start all workers */ /* start all workers */
for (t = 0; t < ctx->threads; t++) { for (t = 0; t < ctx->threads; t++) {
cwork_t *w = &ctx->cwork[t]; cwork_t *w = &ctx->cwork[t];
@@ -394,9 +526,11 @@ void BROTLIMT_freeCCtx(BROTLIMT_CCtx * ctx)
if (!ctx) if (!ctx)
return; return;
pthread_mutex_destroy(&ctx->read_mutex); if (ctx->threads) {
pthread_mutex_destroy(&ctx->write_mutex); pthread_mutex_destroy(&ctx->read_mutex);
free(ctx->cwork); pthread_mutex_destroy(&ctx->write_mutex);
free(ctx->cwork);
}
free(ctx); free(ctx);
ctx = 0; ctx = 0;

120
C/zstdmt/brotli-mt_decompress.c
View File

@@ -50,7 +50,7 @@ struct writelist {
struct BROTLIMT_DCtx_s { struct BROTLIMT_DCtx_s {
/* threads: 1..BROTLIMT_THREAD_MAX */ /* threads: 0, 1..BROTLIMT_THREAD_MAX */
int threads; int threads;
/* should be used for read from input */ /* should be used for read from input */
@@ -96,7 +96,7 @@ BROTLIMT_DCtx *BROTLIMT_createDCtx(int threads, int inputsize)
return 0; return 0;
/* check threads value */ /* check threads value */
if (threads < 1 || threads > BROTLIMT_THREAD_MAX) if (threads < 0 || threads > BROTLIMT_THREAD_MAX)
return 0; return 0;
/* setup ctx */ /* setup ctx */
@@ -112,12 +112,16 @@ BROTLIMT_DCtx *BROTLIMT_createDCtx(int threads, int inputsize)
else else
ctx->inputsize = 1024 * 64; /* 64K buffer */ ctx->inputsize = 1024 * 64; /* 64K buffer */
pthread_mutex_init(&ctx->read_mutex, NULL); if (threads) {
pthread_mutex_init(&ctx->write_mutex, NULL); pthread_mutex_init(&ctx->read_mutex, NULL);
pthread_mutex_init(&ctx->write_mutex, NULL);
INIT_LIST_HEAD(&ctx->writelist_free); INIT_LIST_HEAD(&ctx->writelist_free);
INIT_LIST_HEAD(&ctx->writelist_busy); INIT_LIST_HEAD(&ctx->writelist_busy);
INIT_LIST_HEAD(&ctx->writelist_done); INIT_LIST_HEAD(&ctx->writelist_done);
} else {
threads = 1;
}
ctx->cwork = (cwork_t *) malloc(sizeof(cwork_t) * threads); ctx->cwork = (cwork_t *) malloc(sizeof(cwork_t) * threads);
if (!ctx->cwork) if (!ctx->cwork)
@@ -376,6 +380,90 @@ static void *pt_decompress(void *arg)
return (void *)result; return (void *)result;
} }
/* single threaded (standard brotli stream, without header/mt-frames) */
static size_t st_decompress(void *arg)
{
BROTLIMT_DCtx *ctx = (BROTLIMT_DCtx *) arg;
BrotliDecoderResult bres = BROTLI_DECODER_RESULT_NEEDS_MORE_INPUT;
cwork_t *w = &ctx->cwork[0];
BROTLIMT_Buffer Out;
BROTLIMT_Buffer *out = &Out;
BROTLIMT_Buffer *in = &w->in;
BrotliDecoderState *state;
const uint8_t* next_in;
uint8_t* next_out;
int rv;
size_t retval = 0;
/* allocate space for input buffer */
in->allocated = in->size = ctx->inputsize;
in->buf = malloc(in->size);
if (!in->buf)
return MT_ERROR(memory_allocation);
next_in = in->buf;
/* allocate space for output buffer */
out->allocated = out->size = ctx->inputsize * 4;
out->buf = malloc(out->size);
if (!out->buf) {
free(in->buf);
return MT_ERROR(memory_allocation);
}
next_out = out->buf;
state = BrotliDecoderCreateInstance(NULL, NULL, NULL);
if (!state) {
free(in->buf);
free(out->buf);
return MT_ERROR(memory_allocation);
}
BrotliDecoderSetParameter(state, BROTLI_DECODER_PARAM_LARGE_WINDOW, 1);
while (1) {
if (bres == BROTLI_DECODER_RESULT_NEEDS_MORE_INPUT) {
in->size = in->allocated;
rv = ctx->fn_read(ctx->arg_read, in);
if (in->size == 0) break;
if (rv != 0) {
retval = mt_error(rv);
goto done;
}
next_in = in->buf;
} else if (bres == BROTLI_DECODER_RESULT_NEEDS_MORE_OUTPUT) {
out->size = next_out - (uint8_t*)out->buf;
rv = ctx->fn_write(ctx->arg_write, out);
if (rv != 0) {
retval = mt_error(rv);
goto done;
}
next_out = out->buf;
} else {
break;
}
bres = BrotliDecoderDecompressStream(state, &in->size, &next_in, &out->size, &next_out, 0);
}
if (bres != BROTLI_DECODER_RESULT_SUCCESS) {
retval = MT_ERROR(data_error); // corrupt input
goto done;
}
out->size = next_out - (uint8_t*)out->buf;
if (out->size != 0) {
rv = ctx->fn_write(ctx->arg_write, out);
if (rv != 0) {
retval = mt_error(rv);
goto done;
}
}
done:
free(in->buf);
free(out->buf);
BrotliDecoderDestroyInstance(state);
return retval;
}
size_t BROTLIMT_decompressDCtx(BROTLIMT_DCtx * ctx, BROTLIMT_RdWr_t * rdwr) size_t BROTLIMT_decompressDCtx(BROTLIMT_DCtx * ctx, BROTLIMT_RdWr_t * rdwr)
{ {
unsigned char buf[4]; unsigned char buf[4];
@@ -393,6 +481,14 @@ size_t BROTLIMT_decompressDCtx(BROTLIMT_DCtx * ctx, BROTLIMT_RdWr_t * rdwr)
ctx->arg_read = rdwr->arg_read; ctx->arg_read = rdwr->arg_read;
ctx->arg_write = rdwr->arg_write; ctx->arg_write = rdwr->arg_write;
/* For single threaded brotli (no header, no mt-frames), don't check for
BROTLIMT_MAGIC_SKIPPABLE here, because stdandard brotli stream may also
start with it, so will be mistakenly considered as brotli-mt stream. */
if (ctx->threads == 0) {
/* decompress single threaded */
return st_decompress(ctx);
}
/* check for BROTLIMT_MAGIC_SKIPPABLE */ /* check for BROTLIMT_MAGIC_SKIPPABLE */
in->buf = buf; in->buf = buf;
in->size = 4; in->size = 4;
@@ -402,10 +498,6 @@ size_t BROTLIMT_decompressDCtx(BROTLIMT_DCtx * ctx, BROTLIMT_RdWr_t * rdwr)
if (in->size != 4) if (in->size != 4)
return MT_ERROR(data_error); return MT_ERROR(data_error);
/* single threaded with unknown sizes */
if (MEM_readLE32(buf) != BROTLIMT_MAGIC_SKIPPABLE)
return MT_ERROR(data_error);
/* mark unused */ /* mark unused */
in->buf = 0; in->buf = 0;
in->size = 0; in->size = 0;
@@ -485,8 +577,10 @@ void BROTLIMT_freeDCtx(BROTLIMT_DCtx * ctx)
if (!ctx) if (!ctx)
return; return;
pthread_mutex_destroy(&ctx->read_mutex); if (ctx->threads) {
pthread_mutex_destroy(&ctx->write_mutex); pthread_mutex_destroy(&ctx->read_mutex);
pthread_mutex_destroy(&ctx->write_mutex);
}
free(ctx->cwork); free(ctx->cwork);
free(ctx); free(ctx);
ctx = 0; ctx = 0;

351
CPP/7zip/Archive/BrotliHandler.cpp Normal file
View File

@@ -0,0 +1,351 @@
// BrotliHandler.cpp
#include "StdAfx.h"
#include "../../../C/CpuArch.h"
#include "../../Common/ComTry.h"
#include "../../Common/Defs.h"
#include "../Common/ProgressUtils.h"
#include "../Common/RegisterArc.h"
#include "../Common/StreamUtils.h"
#include "../Compress/BrotliDecoder.h"
#include "../Compress/BrotliEncoder.h"
#include "../Compress/CopyCoder.h"
#include "Common/DummyOutStream.h"
#include "Common/HandlerOut.h"
using namespace NWindows;
namespace NArchive {
namespace NBROTLI {
class CHandler:
public IInArchive,
public IArchiveOpenSeq,
public IOutArchive,
public ISetProperties,
public CMyUnknownImp
{
CMyComPtr<IInStream> _stream;
CMyComPtr<ISequentialInStream> _seqStream;
bool _isArc;
bool _dataAfterEnd;
bool _needMoreInput;
bool _packSize_Defined;
bool _unpackSize_Defined;
UInt64 _packSize;
UInt64 _unpackSize;
UInt64 _numStreams;
UInt64 _numBlocks;
CSingleMethodProps _props;
public:
MY_UNKNOWN_IMP4(
IInArchive,
IArchiveOpenSeq,
IOutArchive,
ISetProperties)
INTERFACE_IInArchive(;)
INTERFACE_IOutArchive(;)
STDMETHOD(OpenSeq)(ISequentialInStream *stream);
STDMETHOD(SetProperties)(const wchar_t * const *names, const PROPVARIANT *values, UInt32 numProps);
CHandler() { }
};
static const Byte kProps[] =
{
kpidSize,
kpidPackSize
};
static const Byte kArcProps[] =
{
kpidNumStreams,
kpidNumBlocks
};
IMP_IInArchive_Props
IMP_IInArchive_ArcProps
STDMETHODIMP CHandler::GetArchiveProperty(PROPID /*propID*/, PROPVARIANT * /*value*/)
{
return S_OK;
}
STDMETHODIMP CHandler::GetNumberOfItems(UInt32 *numItems)
{
*numItems = 1;
return S_OK;
}
STDMETHODIMP CHandler::GetProperty(UInt32 /* index */, PROPID propID, PROPVARIANT *value)
{
NCOM::CPropVariant prop;
switch (propID)
{
case kpidPackSize: if (_packSize_Defined) prop = _packSize; break;
case kpidSize: if (_unpackSize_Defined) prop = _unpackSize; break;
}
prop.Detach(value);
return S_OK;
}
// brotli format has no signature
// brotli stream can't be veriefied unless errors by unpack
#if 0
API_FUNC_static_IsArc IsArc_Brotli(const Byte *p, size_t size)
{
return k_IsArc_Res_YES;
}
}
#endif
STDMETHODIMP CHandler::Open(IInStream *stream, const UInt64 *, IArchiveOpenCallback *)
{
COM_TRY_BEGIN
Close();
{
_isArc = true;
_stream = stream;
_seqStream = stream;
}
return S_OK;
COM_TRY_END
}
STDMETHODIMP CHandler::OpenSeq(ISequentialInStream *stream)
{
Close();
_isArc = true;
_seqStream = stream;
return S_OK;
}
STDMETHODIMP CHandler::Close()
{
_isArc = false;
_dataAfterEnd = false;
_needMoreInput = false;
_packSize_Defined = false;
_unpackSize_Defined = false;
_packSize = 0;
_seqStream.Release();
_stream.Release();
return S_OK;
}
STDMETHODIMP CHandler::Extract(const UInt32 *indices, UInt32 numItems,
Int32 testMode, IArchiveExtractCallback *extractCallback)
{
COM_TRY_BEGIN
if (numItems == 0)
return S_OK;
if (numItems != (UInt32)(Int32)-1 && (numItems != 1 || indices[0] != 0))
return E_INVALIDARG;
if (_packSize_Defined)
extractCallback->SetTotal(_packSize);
CMyComPtr<ISequentialOutStream> realOutStream;
Int32 askMode = testMode ?
NExtract::NAskMode::kTest :
NExtract::NAskMode::kExtract;
RINOK(extractCallback->GetStream(0, &realOutStream, askMode));
if (!testMode && !realOutStream)
return S_OK;
extractCallback->PrepareOperation(askMode);
Int32 opRes;
{
NCompress::NBROTLI::CDecoder *decoderSpec = new NCompress::NBROTLI::CDecoder;
decoderSpec->SetNumberOfThreads(0); /* .br - single threaded processing (without header/mt-frames) */
CMyComPtr<ICompressCoder> decoder = decoderSpec;
decoderSpec->SetInStream(_seqStream);
CDummyOutStream *outStreamSpec = new CDummyOutStream;
CMyComPtr<ISequentialOutStream> outStream(outStreamSpec);
outStreamSpec->SetStream(realOutStream);
outStreamSpec->Init();
realOutStream.Release();
CLocalProgress *lps = new CLocalProgress;
CMyComPtr<ICompressProgressInfo> progress = lps;
lps->Init(extractCallback, true);
UInt64 packSize = 0;
UInt64 unpackedSize = 0;
HRESULT result = S_OK;
for (;;)
{
lps->InSize = packSize;
lps->OutSize = unpackedSize;
RINOK(lps->SetCur());
result = decoderSpec->CodeResume(outStream, &unpackedSize, progress);
UInt64 streamSize = decoderSpec->GetInputProcessedSize();
if (result != S_FALSE && result != S_OK)
return result;
if (unpackedSize == 0)
break;
if (streamSize == packSize)
{
// no new bytes in input stream, So it's good end of archive.
result = S_OK;
break;
}
if (packSize > streamSize)
return E_FAIL;
if (result != S_OK)
break;
}
decoderSpec->ReleaseInStream();
outStream.Release();
if (!_isArc)
opRes = NExtract::NOperationResult::kIsNotArc;
else if (_needMoreInput)
opRes = NExtract::NOperationResult::kUnexpectedEnd;
else if (_dataAfterEnd)
opRes = NExtract::NOperationResult::kDataAfterEnd;
else if (result == S_FALSE)
opRes = NExtract::NOperationResult::kDataError;
else if (result == S_OK)
opRes = NExtract::NOperationResult::kOK;
else
return result;
}
return extractCallback->SetOperationResult(opRes);
COM_TRY_END
}
static HRESULT UpdateArchive(
UInt64 unpackSize,
ISequentialOutStream *outStream,
const CProps &props,
IArchiveUpdateCallback *updateCallback)
{
RINOK(updateCallback->SetTotal(unpackSize));
CMyComPtr<ISequentialInStream> fileInStream;
RINOK(updateCallback->GetStream(0, &fileInStream));
CLocalProgress *localProgressSpec = new CLocalProgress;
CMyComPtr<ICompressProgressInfo> localProgress = localProgressSpec;
localProgressSpec->Init(updateCallback, true);
NCompress::NBROTLI::CEncoder *encoderSpec = new NCompress::NBROTLI::CEncoder;
encoderSpec->unpackSize = unpackSize;
encoderSpec->SetNumberOfThreads(0); /* .br - single threaded processing (without header/mt-frames) */
CMyComPtr<ICompressCoder> encoder = encoderSpec;
RINOK(props.SetCoderProps(encoderSpec, NULL));
RINOK(encoder->Code(fileInStream, outStream, NULL, NULL, localProgress));
return updateCallback->SetOperationResult(NArchive::NUpdate::NOperationResult::kOK);
}
STDMETHODIMP CHandler::GetFileTimeType(UInt32 *type)
{
*type = NFileTimeType::kUnix;
return S_OK;
}
STDMETHODIMP CHandler::UpdateItems(ISequentialOutStream *outStream, UInt32 numItems,
IArchiveUpdateCallback *updateCallback)
{
COM_TRY_BEGIN
if (numItems != 1)
return E_INVALIDARG;
Int32 newData, newProps;
UInt32 indexInArchive;
if (!updateCallback)
return E_FAIL;
RINOK(updateCallback->GetUpdateItemInfo(0, &newData, &newProps, &indexInArchive));
if ((newProps))
{
{
NCOM::CPropVariant prop;
RINOK(updateCallback->GetProperty(0, kpidIsDir, &prop));
if (prop.vt != VT_EMPTY)
if (prop.vt != VT_BOOL || prop.boolVal != VARIANT_FALSE)
return E_INVALIDARG;
}
}
if ((newData))
{
UInt64 size;
{
NCOM::CPropVariant prop;
RINOK(updateCallback->GetProperty(0, kpidSize, &prop));
if (prop.vt != VT_UI8)
return E_INVALIDARG;
size = prop.uhVal.QuadPart;
}
return UpdateArchive(size, outStream, _props, updateCallback);
}
if (indexInArchive != 0)
return E_INVALIDARG;
CLocalProgress *lps = new CLocalProgress;
CMyComPtr<ICompressProgressInfo> progress = lps;
lps->Init(updateCallback, true);
CMyComPtr<IArchiveUpdateCallbackFile> opCallback;
updateCallback->QueryInterface(IID_IArchiveUpdateCallbackFile, (void **)&opCallback);
if (opCallback)
{
RINOK(opCallback->ReportOperation(
NEventIndexType::kInArcIndex, 0,
NUpdateNotifyOp::kReplicate))
}
if (_stream)
RINOK(_stream->Seek(0, STREAM_SEEK_SET, NULL));
return NCompress::CopyStream(_stream, outStream, progress);
COM_TRY_END
}
STDMETHODIMP CHandler::SetProperties(const wchar_t * const *names, const PROPVARIANT *values, UInt32 numProps)
{
return _props.SetProperties(names, values, numProps);
}
IMP_CreateArcIn
IMP_CreateArcOut
REGISTER_ARC_R(
"brotli", "br brotli tbr", 0, 0x1F,
0, NULL,
0,
NArcInfoFlags::kKeepName | NArcInfoFlags::kPureStartOpen | NArcInfoFlags::kByExtOnlyOpen,
0,
CreateArc, CreateArcOut,
/* IsArc_Brotli */)
}}

1
CPP/7zip/Bundles/Format7zF/Arc.mak
View File

@@ -70,6 +70,7 @@ AR_OBJS = \
$O\ArHandler.obj \ $O\ArHandler.obj \
$O\ArjHandler.obj \ $O\ArjHandler.obj \
$O\Base64Handler.obj \ $O\Base64Handler.obj \
$O\BrotliHandler.obj \
$O\Bz2Handler.obj \ $O\Bz2Handler.obj \
$O\ComHandler.obj \ $O\ComHandler.obj \
$O\CpioHandler.obj \ $O\CpioHandler.obj \

2
CPP/7zip/Compress/BrotliDecoder.cpp
View File

@@ -98,7 +98,7 @@ STDMETHODIMP CDecoder::SetDecoderProperties2(const Byte * prop, UInt32 size)
STDMETHODIMP CDecoder::SetNumberOfThreads(UInt32 numThreads) STDMETHODIMP CDecoder::SetNumberOfThreads(UInt32 numThreads)
{ {
const UInt32 kNumThreadsMax = BROTLIMT_THREAD_MAX; const UInt32 kNumThreadsMax = BROTLIMT_THREAD_MAX;
if (numThreads < 1) numThreads = 1; if (numThreads < 0) numThreads = 0;
if (numThreads > kNumThreadsMax) numThreads = kNumThreadsMax; if (numThreads > kNumThreadsMax) numThreads = kNumThreadsMax;
_numThreads = numThreads; _numThreads = numThreads;
return S_OK; return S_OK;

44
CPP/7zip/Compress/BrotliEncoder.cpp
View File

@@ -12,8 +12,10 @@ CEncoder::CEncoder():
_processedIn(0), _processedIn(0),
_processedOut(0), _processedOut(0),
_inputSize(0), _inputSize(0),
_ctx(NULL), _numThreads(NWindows::NSystem::GetNumberOfProcessors()),
_numThreads(NWindows::NSystem::GetNumberOfProcessors()) _Long(-1),
_WindowLog(-1),
_ctx(NULL)
{ {
_props.clear(); _props.clear();
} }
@@ -52,6 +54,37 @@ STDMETHODIMP CEncoder::SetCoderProperties(const PROPID * propIDs, const PROPVARI
SetNumberOfThreads(v); SetNumberOfThreads(v);
break; break;
} }
case NCoderPropID::kLong:
{
/* like --long in zstd cli program */
_Long = 1;
if (v == 0) {
if (prop.vt == VT_EMPTY) {
// m0=brotli:long -> long=default
_WindowLog = BROTLI_MAX_WINDOW_BITS; //BROTLI_DEFAULT_WINDOW;
} else {
// m0=brotli:long=0 -> long=auto
_WindowLog = 0;
}
} else if (v < BROTLI_MIN_WINDOW_BITS) {
// m0=brotli:long=9 -> long=10
_WindowLog = BROTLI_MIN_WINDOW_BITS;
} else if (v > BROTLI_LARGE_MAX_WINDOW_BITS) {
// m0=brotli:long=33 -> long=max
_WindowLog = BROTLI_LARGE_MAX_WINDOW_BITS;
} else {
// m0=brotli:long=15 -> long=value
_WindowLog = v;
}
break;
}
case NCoderPropID::kWindowLog:
{
if (v < BROTLI_MIN_WINDOW_BITS) v = BROTLI_MIN_WINDOW_BITS;
if (v > BROTLI_MAX_WINDOW_BITS) v = BROTLI_MAX_WINDOW_BITS;
_WindowLog = v;
break;
}
default: default:
{ {
break; break;
@@ -102,11 +135,12 @@ STDMETHODIMP CEncoder::Code(ISequentialInStream *inStream,
/* 2) create compression context, if needed */ /* 2) create compression context, if needed */
if (!_ctx) if (!_ctx)
_ctx = BROTLIMT_createCCtx(_numThreads, _props._level, _inputSize); _ctx = BROTLIMT_createCCtx(_numThreads, unpackSize, _props._level, _inputSize,
_WindowLog >= 0 ? _WindowLog : BROTLI_MAX_WINDOW_BITS);
if (!_ctx) if (!_ctx)
return S_FALSE; return S_FALSE;
/* 3) compress */ /* 4) compress */
result = BROTLIMT_compressCCtx(_ctx, &rdwr); result = BROTLIMT_compressCCtx(_ctx, &rdwr);
if (BROTLIMT_isError(result)) { if (BROTLIMT_isError(result)) {
if (result == (size_t)-BROTLIMT_error_canceled) if (result == (size_t)-BROTLIMT_error_canceled)
@@ -120,7 +154,7 @@ STDMETHODIMP CEncoder::Code(ISequentialInStream *inStream,
STDMETHODIMP CEncoder::SetNumberOfThreads(UInt32 numThreads) STDMETHODIMP CEncoder::SetNumberOfThreads(UInt32 numThreads)
{ {
const UInt32 kNumThreadsMax = BROTLIMT_THREAD_MAX; const UInt32 kNumThreadsMax = BROTLIMT_THREAD_MAX;
if (numThreads < 1) numThreads = 1; if (numThreads < 0) numThreads = 0;
if (numThreads > kNumThreadsMax) numThreads = kNumThreadsMax; if (numThreads > kNumThreadsMax) numThreads = kNumThreadsMax;
_numThreads = numThreads; _numThreads = numThreads;
return S_OK; return S_OK;

6
CPP/7zip/Compress/BrotliEncoder.h
View File

@@ -45,9 +45,15 @@ class CEncoder:
UInt32 _inputSize; UInt32 _inputSize;
UInt32 _numThreads; UInt32 _numThreads;
Int32 _Long;
Int32 _WindowLog;
BROTLIMT_CCtx *_ctx; BROTLIMT_CCtx *_ctx;
public: public:
UInt64 unpackSize;
MY_QUERYINTERFACE_BEGIN2(ICompressCoder) MY_QUERYINTERFACE_BEGIN2(ICompressCoder)
MY_QUERYINTERFACE_ENTRY(ICompressSetCoderMt) MY_QUERYINTERFACE_ENTRY(ICompressSetCoderMt)
MY_QUERYINTERFACE_ENTRY(ICompressSetCoderProperties) MY_QUERYINTERFACE_ENTRY(ICompressSetCoderProperties)