From 71a0a67ad63ae1ff5757d9ab3d9c26307b4bab46 Mon Sep 17 00:00:00 2001 From: Tino Reichardt Date: Sun, 13 Dec 2020 01:27:42 +0100 Subject: [PATCH] Fix extracting multiple LZ4 frames --- C/7zVersion.h | 4 +- C/zstdmt/lz4-mt_decompress.c | 80 ++++++++++++++++++------------------ README.md | 7 ++-- 3 files changed, 45 insertions(+), 46 deletions(-) diff --git a/C/7zVersion.h b/C/7zVersion.h index 33c72a8c..ee4f272f 100644 --- a/C/7zVersion.h +++ b/C/7zVersion.h @@ -1,7 +1,7 @@ #define MY_VER_MAJOR 19 #define MY_VER_MINOR 00 #define MY_VER_BUILD 0 -#define MY_VERSION_NUMBERS "19.00 ZS v1.4.5 R4" +#define MY_VERSION_NUMBERS "19.00 ZS v1.4.5 R5" #define MY_VERSION MY_VERSION_NUMBERS #ifdef MY_CPU_NAME @@ -10,7 +10,7 @@ #define MY_VERSION_CPU MY_VERSION #endif -#define MY_DATE "2020-12-06" +#define MY_DATE "2020-12-19" #undef MY_COPYRIGHT #undef MY_VERSION_COPYRIGHT_DATE #define MY_AUTHOR_NAME "Igor Pavlov, Tino Reichardt" diff --git a/C/zstdmt/lz4-mt_decompress.c b/C/zstdmt/lz4-mt_decompress.c index e41e149a..fb63b799 100644 --- a/C/zstdmt/lz4-mt_decompress.c +++ b/C/zstdmt/lz4-mt_decompress.c @@ -1,6 +1,6 @@ /** - * Copyright (c) 2016 - 2017 Tino Reichardt + * Copyright (c) 2016 - 2020 Tino Reichardt * All rights reserved. * * This source code is licensed under the BSD-style license found in the @@ -112,7 +112,7 @@ LZ4MT_DCtx *LZ4MT_createDCtx(int threads, int inputsize) if (inputsize) ctx->inputsize = inputsize; else - ctx->inputsize = 1024 * 64; /* 64K buffer */ + ctx->inputsize = 1024 + 1024 * 4; pthread_mutex_init(&ctx->read_mutex, NULL); pthread_mutex_init(&ctx->write_mutex, NULL); @@ -391,13 +391,12 @@ static void *pt_decompress(void *arg) static size_t st_decompress(void *arg) { LZ4MT_DCtx *ctx = (LZ4MT_DCtx *) arg; - LZ4F_errorCode_t nextToLoad = 0; + LZ4F_errorCode_t result = 0; cwork_t *w = &ctx->cwork[0]; LZ4MT_Buffer Out; LZ4MT_Buffer *out = &Out; LZ4MT_Buffer *in = &w->in; void *magic = in->buf; - size_t pos = 0; int rv; /* allocate space for input buffer */ @@ -418,47 +417,29 @@ static size_t st_decompress(void *arg) in->size = 4; memcpy(in->buf, magic, in->size); - nextToLoad = - LZ4F_decompress(w->dctx, out->buf, &pos, in->buf, &in->size, 0); - if (LZ4F_isError(nextToLoad)) { - free(in->buf); - free(out->buf); - return ERROR(compression_library); - } + /* stats */ + ctx->insize = 4; + ctx->outsize = 0; - for (; nextToLoad; pos = 0) { - if (nextToLoad > ctx->inputsize) - nextToLoad = ctx->inputsize; - - /* read new input */ - in->size = nextToLoad; - rv = ctx->fn_read(ctx->arg_read, in); - if (rv != 0) { - free(in->buf); - free(out->buf); - return mt_error(rv); - } - - /* done, eof reached */ - if (in->size == 0) - break; - - /* still to read, or still to flush */ - while ((pos < in->size) || (out->size == ctx->inputsize)) { - size_t remaining = in->size - pos; + /* decompress loop */ + for (;;) { + size_t srcPos = 0; + for (;;) { + size_t srcSize = in->size - srcPos; out->size = ctx->inputsize; - /* decompress */ - nextToLoad = - LZ4F_decompress(w->dctx, out->buf, &out->size, - (unsigned char *)in->buf + pos, - &remaining, NULL); - if (LZ4F_isError(nextToLoad)) { + result = LZ4F_decompress(w->dctx, out->buf, &out->size, (unsigned char *)in->buf + srcPos, &srcSize, NULL); + if (LZ4F_isError(result)) { free(in->buf); free(out->buf); return ERROR(compression_library); } + /* update stats */ + srcPos += srcSize; + ctx->insize += srcSize; + ctx->outsize += out->size; + /* have some output */ if (out->size) { rv = ctx->fn_write(ctx->arg_write, out); @@ -469,11 +450,30 @@ static size_t st_decompress(void *arg) } } - if (nextToLoad == 0) + /* consumed all input */ + if (srcPos == in->size) break; - - pos += remaining; } + + /* read new input */ + if (result) + in->size = result; + else + in->size = ctx->inputsize; + + if (in->size > ctx->inputsize) + in->size = ctx->inputsize; + + rv = ctx->fn_read(ctx->arg_read, in); + ctx->insize += in->size; + if (rv != 0) { + free(in->buf); + free(out->buf); + return mt_error(rv); + } + + if (in->size == 0) + break; } /* no error */ diff --git a/README.md b/README.md index 9cdf750b..863b2841 100644 --- a/README.md +++ b/README.md @@ -46,7 +46,7 @@ You can install it in two ways: The output should look like this: ``` -7-Zip 19.00 ZS v1.4.5 R4 (x64) : Copyright (c) 1999-2019 Igor Pavlov, 2016-2020 Tino Reichardt : 2020-12-06 +7-Zip 19.00 ZS v1.4.5 R4 (x64) : Copyright (c) 1999-2019 Igor Pavlov, 2016-2020 Tino Reichardt : 2020-12-17 Libs: 0 c:\Program Files\7-Zip-Zstandard\7z.dll @@ -267,7 +267,7 @@ For the benchmarks I am using Windows 7 64bit on my Laptop which has the followi - [7-Zip Zstandard Homepage](https://mcmilk.de/projects/7-Zip-zstd/) - [Request for inclusion](https://sourceforge.net/p/sevenzip/discussion/45797/thread/a7e4f3f3/) into the mainline 7-Zip: - result, will currently not included :( -- [p7zip Homepage with Zstd](https://github.com/jinfeihan57/p7zip) +- [p7zip Homepage](https://github.com/jinfeihan57/p7zip) - for Linux and MacOS with LZ4 and Zstandard ## Donate @@ -285,7 +285,7 @@ You find this project useful, maybe you consider a donation ;-) - [LZ5] Version 1.5 - [Zstandard] Version 1.4.5 -/TR 2020-12-06 +/TR 2020-12-17 ## Notes @@ -300,6 +300,5 @@ You find this project useful, maybe you consider a donation ;-) [Lizard]:https://github.com/inikep/lizard/ [ImDisk]:https://sourceforge.net/projects/imdisk-toolkit/ [Fast LZMA2]:https://github.com/conor42/fast-lzma2 - [Codecs.7z]:https://github.com/mcmilk/7-Zip-zstd/releases [TotalCmd.7z]:https://github.com/mcmilk/7-Zip-zstd/releases