xevion/easy7zip
1
0
Fork 0
mirror of https://github.com/Xevion/easy7zip.git synced 2025-12-11 06:07:12 -06:00
Code Issues Packages Projects Releases Wiki Activity

Major changes, including Brotli and Lizard

Browse Source 
- update of zstd-mt library
- add brotli v0.6.0
- add lizard v2.0
- xxhash is from zstd for lz4, lz5 and lizard now
- update also the documentation, where needed
This commit is contained in:
Tino Reichardt
2017-05-25 18:40:15 +02:00
parent 40e87f615c
commit 5ff0657d9f
173 changed files with 3936 additions and 6591 deletions
Download Patch File Download Diff File Expand all files Collapse all files

24
C/zstdmt/LICENSE Normal file
View File

@@ -0,0 +1,24 @@
BSD License
Copyright (c) 2016 - 2017, Tino Reichardt, All rights reserved.
Redistribution and use in source and binary forms, with or without modification,
are permitted provided that the following conditions are met:
* Redistributions of source code must retain the above copyright notice, this
list of conditions and the following disclaimer.
* Redistributions in binary form must reproduce the above copyright notice,
this list of conditions and the following disclaimer in the documentation
and/or other materials provided with the distribution.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR
ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

50
C/zstdmt/README.md
View File

@@ -1,8 +1,48 @@
# Multithreading Library for [LZ4], [LZ5] and [ZStandard]
# Multithreading Library for [Brotli], [Lizard], [LZ4], [LZ5] and [ZStandard]
- this library is used as a threading wrapper for zstd, lz4 and lz5
- zstandard has it's own now, maybe I should switch to it
- homepage of zstdmt is: https://github.com/mcmilk/zstdmt
## Description
- works with skippables frame id 0x184D2A50 (12 bytes per compressed frame)
- brotli is supported the same way, it will encapsulate the real brotli stream
within an 16 byte frame header
/TR 2017-05-17
## Generic skippable frame definition
- the frame header for [Lizard], [LZ4], [LZ5] and [ZStandard] is like this:
size | value | description
--------|-------------------|------------
4 bytes | 0x184D2A50U | magic for skippable frame
4 bytes | 4 | size of skippable frame
4 bytes | compressed size | size of the following frame (compressed data)
## [Brotli] frame definition
- the frame header for brotli is defined a bit different:
size | value | description
--------|-------------------|------------
4 bytes | 0x184D2A50U | magic for skippable frame (like zstd)
4 bytes | 8 | size of skippable frame
4 bytes | compressed size | size of the following frame (compressed data)
2 bytes | 0x5242U | magic for brotli "BR"
2 bytes | uncompressed size | allocation hint for decompressor (64KB * this size)
## Usage of the Testutils
- see [programs](https://github.com/mcmilk/zstdmt/tree/master/programs)
## Usage of the Library
- see [lib](https://github.com/mcmilk/zstdmt/tree/master/lib)
[Brotli]:https://github.com/google/brotli/
[LZ4]:https://github.com/lz4/lz4/
[LZ5]:https://github.com/inikep/lz5/
[ZStandard]:https://github.com/facebook/zstd/
[Lizard]:https://github.com/inikep/lizard/
/TR 2017-05-24

15
C/zstdmt/brotli-mt.h
View File

@@ -26,17 +26,19 @@ extern "C" {
/* current maximum the library will accept */
#define BROTLIMT_THREAD_MAX 128
#define BROTLIMT_LEVEL_MIN 0
#define BROTLIMT_LEVEL_MAX 11
#define BROFMT_MAGICNUMBER 0x544F5242U /* BROT */
#define BROFMT_MAGIC_SKIPPABLE 0x184D2A50U
#define BROTLIMT_MAGICNUMBER 0x5242U /* BR */
#define BROTLIMT_MAGIC_SKIPPABLE 0x184D2A50U
#define BROTLI_VERSION_MAJOR 0
#define BROTLI_VERSION_MINOR 6
/* **************************************
* Error Handling
****************************************/
extern size_t bromt_errcode;
typedef enum {
BROTLIMT_error_no_error,
BROTLIMT_error_memory_allocation,
@@ -51,11 +53,8 @@ typedef enum {
BROTLIMT_error_maxCode
} BROTLIMT_ErrorCode;
#ifdef ERROR
# undef ERROR /* reported already defined on VS 2015 (Rich Geldreich) */
#endif
#define PREFIX(name) BROTLIMT_error_##name
#define ERROR(name) ((size_t)-PREFIX(name))
#define MT_ERROR(name) ((size_t)-PREFIX(name))
extern unsigned BROTLIMT_isError(size_t code);
extern const char* BROTLIMT_getErrorString(size_t code);

28
C/zstdmt/brotli-mt_common.c
View File

@@ -1,7 +1,7 @@
/**
* Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
* Copyright (c) 2016 Tino Reichardt
* Copyright (c) 2016 - 2017 Tino Reichardt
* All rights reserved.
*
* This source code is licensed under the BSD-style license found in the
@@ -9,14 +9,11 @@
* of patent rights can be found in the PATENTS file in the same directory.
*/
#include <brotli/encode.h>
#include <brotli/decode.h>
#include "encode.h"
#include "decode.h"
#include "brotli-mt.h"
/* will be used for lib errors */
size_t bromt_errcode;
/* ****************************************
* BROMT Error Management
******************************************/
@@ -26,23 +23,15 @@ size_t bromt_errcode;
*/
unsigned BROTLIMT_isError(size_t code)
{
return (code > ERROR(maxCode));
return (code > MT_ERROR(maxCode));
}
/**
* BROTLIMT_getErrorString() - give error code string from function result
* BROTLIMT_getErrorString() - give our error code string of result
*/
const char *BROTLIMT_getErrorString(size_t code)
{
if (code > 10)
return "nono";
return "nono2";
#if 0
if (BROF_isError(bromt_errcode))
return BROF_getErrorName(bromt_errcode);
static const char *notErrorCode = "Unspecified error bromt code";
static const char *noErrorCode = "Unspecified brotli error code";
switch ((BROTLIMT_ErrorCode) (0 - code)) {
case PREFIX(no_error):
return "No error detected";
@@ -60,11 +49,8 @@ const char *BROTLIMT_getErrorString(size_t code)
return "Could not decompress frame at once";
case PREFIX(compressionParameter_unsupported):
return "Compression parameter is out of bound";
case PREFIX(compression_library):
return "Compression library reports failure";
case PREFIX(maxCode):
default:
return notErrorCode;
return noErrorCode;
}
#endif
}

62
C/zstdmt/brotli-mt_compress.c
View File

@@ -13,9 +13,8 @@
#include <stdlib.h>
#include <string.h>
#include <stdio.h>
#include <brotli/encode.h>
#include "encode.h"
#include "brotli-mt.h"
#include "memmt.h"
@@ -23,7 +22,7 @@
#include "list.h"
/**
* multi threaded bro - multiple workers version
* multi threaded brotli - multiple workers version
*
* - each thread works on his own
* - no main thread which does reading and then starting the work
@@ -49,8 +48,6 @@ struct writelist {
};
struct BROTLIMT_CCtx_s {
/* level: 1..22 */
int level;
/* threads: 1..BROTLIMT_THREAD_MAX */
@@ -103,14 +100,14 @@ BROTLIMT_CCtx *BROTLIMT_createCCtx(int threads, int level, int inputsize)
return 0;
/* check level */
if (level < 1 || level > BROTLIMT_LEVEL_MAX)
if (level < BROTLIMT_LEVEL_MIN || level > BROTLIMT_LEVEL_MAX)
return 0;
/* calculate chunksize for one thread */
if (inputsize)
ctx->inputsize = inputsize;
else
ctx->inputsize = 1024 * 1024 * level;
ctx->inputsize = 1024 * 1024 * (level ? level : 1);
/* setup ctx */
ctx->level = level;
@@ -152,15 +149,14 @@ static size_t mt_error(int rv)
{
switch (rv) {
case -1:
return ERROR(read_fail);
return MT_ERROR(read_fail);
case -2:
return ERROR(canceled);
return MT_ERROR(canceled);
case -3:
return ERROR(memory_allocation);
return MT_ERROR(memory_allocation);
}
/* XXX, some catch all other errors */
return ERROR(read_fail);
return MT_ERROR(read_fail);
}
/**
@@ -206,7 +202,7 @@ static void *pt_compress(void *arg)
in.size = ctx->inputsize;
in.buf = malloc(in.size);
if (!in.buf)
return (void *)ERROR(memory_allocation);
return (void *)MT_ERROR(memory_allocation);
for (;;) {
struct list_head *entry;
@@ -228,14 +224,14 @@ static void *pt_compress(void *arg)
malloc(sizeof(struct writelist));
if (!wl) {
pthread_mutex_unlock(&ctx->write_mutex);
return (void *)ERROR(memory_allocation);
return (void *)MT_ERROR(memory_allocation);
}
wl->out.size =
BrotliEncoderMaxCompressedSize(ctx->inputsize) + 16;
wl->out.buf = malloc(wl->out.size);
if (!wl->out.buf) {
pthread_mutex_unlock(&ctx->write_mutex);
return (void *)ERROR(memory_allocation);
return (void *)MT_ERROR(memory_allocation);
}
list_add(&wl->node, &ctx->writelist_busy);
}
@@ -251,7 +247,7 @@ static void *pt_compress(void *arg)
}
/* eof */
if (in.size == 0) {
if (in.size == 0 && ctx->frames > 0) {
free(in.buf);
pthread_mutex_unlock(&ctx->read_mutex);
@@ -268,31 +264,45 @@ static void *pt_compress(void *arg)
/* compress whole frame */
{
const uint8_t *ibuf = in.buf;
uint8_t *obuf = wl->out.buf + 12;
wl->out.size -= 12;
uint8_t *obuf = (uint8_t*)wl->out.buf + 16;
wl->out.size -= 16;
rv = BrotliEncoderCompress(ctx->level,
BROTLI_MAX_WINDOW_BITS,
BROTLI_MODE_GENERIC, in.size,
ibuf, &wl->out.size, obuf);
/* printf("BrotliEncoderCompress() rv=%d in=%zu out=%zu\n", rv, in.size, wl->out.size); */
if (rv == BROTLI_FALSE) {
pthread_mutex_lock(&ctx->write_mutex);
list_move(&wl->node, &ctx->writelist_free);
pthread_mutex_unlock(&ctx->write_mutex);
/* user can lookup that code */
//bromt_errcode = result;
return (void *)-1;
return (void *)MT_ERROR(frame_compress);
}
}
/* write skippable frame */
MEM_writeLE32((unsigned char *)wl->out.buf + 0,
BROFMT_MAGIC_SKIPPABLE);
MEM_writeLE32((unsigned char *)wl->out.buf + 4, 4);
BROTLIMT_MAGIC_SKIPPABLE);
MEM_writeLE32((unsigned char *)wl->out.buf + 4, 8);
MEM_writeLE32((unsigned char *)wl->out.buf + 8,
(U32) wl->out.size);
MEM_writeLE32((unsigned char *)wl->out.buf + 12,
(U32) BROFMT_MAGICNUMBER);
/* BR */
MEM_writeLE16((unsigned char *)wl->out.buf + 12,
(U16) BROTLIMT_MAGICNUMBER);
/* number of 64KB blocks needed for decompression */
{
U16 hintsize;
if (ctx->inputsize > (int)in.size) {
hintsize = (U16)(in.size >> 16);
hintsize += 1;
} else
hintsize = ctx->inputsize >> 16;
MEM_writeLE16((unsigned char *)wl->out.buf + 14,
hintsize);
}
wl->out.size += 16;
/* write result */
@@ -313,7 +323,7 @@ size_t BROTLIMT_compressCCtx(BROTLIMT_CCtx * ctx, BROTLIMT_RdWr_t * rdwr)
void *retval_of_thread = 0;
if (!ctx)
return ERROR(compressionParameter_unsupported);
return MT_ERROR(compressionParameter_unsupported);
/* init reading and writing functions */
ctx->fn_read = rdwr->fn_read;

204
C/zstdmt/brotli-mt_decompress.c
View File

@@ -14,7 +14,7 @@
#include <stdlib.h>
#include <string.h>
#include <brotli/decode.h>
#include "decode.h"
#include "brotli-mt.h"
#include "memmt.h"
@@ -22,7 +22,7 @@
#include "list.h"
/**
* multi threaded bro - multiple workers version
* multi threaded brotli - multiple workers version
*
* - each thread works on his own
* - no main thread which does reading and then starting the work
@@ -143,15 +143,15 @@ static size_t mt_error(int rv)
{
switch (rv) {
case -1:
return ERROR(read_fail);
return MT_ERROR(read_fail);
case -2:
return ERROR(canceled);
return MT_ERROR(canceled);
case -3:
return ERROR(memory_allocation);
return MT_ERROR(memory_allocation);
}
/* XXX, some catch all other errors */
return ERROR(read_fail);
return MT_ERROR(read_fail);
}
/**
@@ -184,30 +184,30 @@ static size_t pt_write(BROTLIMT_DCtx * ctx, struct writelist *wl)
/**
* pt_read - read compressed output
*/
static size_t pt_read(BROTLIMT_DCtx * ctx, BROTLIMT_Buffer * in, size_t * frame)
static size_t pt_read(BROTLIMT_DCtx * ctx, BROTLIMT_Buffer * in, size_t * frame, size_t * uncompressed)
{
unsigned char hdrbuf[12];
unsigned char hdrbuf[16];
BROTLIMT_Buffer hdr;
int rv;
/* read skippable frame (8 or 12 bytes) */
/* read skippable frame (12 or 16 bytes) */
pthread_mutex_lock(&ctx->read_mutex);
/* special case, first 4 bytes already read */
if (ctx->frames == 0) {
hdr.buf = hdrbuf + 4;
hdr.size = 8;
hdr.size = 12;
rv = ctx->fn_read(ctx->arg_read, &hdr);
if (rv != 0) {
pthread_mutex_unlock(&ctx->read_mutex);
return mt_error(rv);
}
if (hdr.size != 8)
if (hdr.size != 12)
goto error_read;
hdr.buf = hdrbuf;
} else {
hdr.buf = hdrbuf;
hdr.size = 12;
hdr.size = 16;
rv = ctx->fn_read(ctx->arg_read, &hdr);
if (rv != 0) {
pthread_mutex_unlock(&ctx->read_mutex);
@@ -219,18 +219,26 @@ static size_t pt_read(BROTLIMT_DCtx * ctx, BROTLIMT_Buffer * in, size_t * frame)
in->size = 0;
return 0;
}
if (hdr.size != 12)
if (hdr.size != 16)
goto error_read;
if (MEM_readLE32((unsigned char *)hdr.buf + 0) !=
BROFMT_MAGIC_SKIPPABLE)
BROTLIMT_MAGIC_SKIPPABLE)
goto error_data;
}
/* check header data */
if (MEM_readLE32((unsigned char *)hdr.buf + 4) != 4)
if (MEM_readLE32((unsigned char *)hdr.buf + 4) != 8)
goto error_data;
if (MEM_readLE16((unsigned char *)hdr.buf + 12) != BROTLIMT_MAGICNUMBER)
goto error_data;
ctx->insize += 12;
/* get uncompressed size for output buffer */
{
U16 hintsize = MEM_readLE16((unsigned char *)hdr.buf + 14);
*uncompressed = hintsize << 16;
}
ctx->insize += 16;
/* read new inputsize */
{
size_t toRead = MEM_readLE32((unsigned char *)hdr.buf + 8);
@@ -266,13 +274,13 @@ static size_t pt_read(BROTLIMT_DCtx * ctx, BROTLIMT_Buffer * in, size_t * frame)
error_data:
pthread_mutex_unlock(&ctx->read_mutex);
return ERROR(data_error);
return MT_ERROR(data_error);
error_read:
pthread_mutex_unlock(&ctx->read_mutex);
return ERROR(read_fail);
return MT_ERROR(read_fail);
error_nomem:
pthread_mutex_unlock(&ctx->read_mutex);
return ERROR(memory_allocation);
return MT_ERROR(memory_allocation);
}
static void *pt_decompress(void *arg)
@@ -286,6 +294,7 @@ static void *pt_decompress(void *arg)
for (;;) {
struct list_head *entry;
BROTLIMT_Buffer *out;
int rv;
/* allocate space for new output */
pthread_mutex_lock(&ctx->write_mutex);
@@ -299,7 +308,7 @@ static void *pt_decompress(void *arg)
wl = (struct writelist *)
malloc(sizeof(struct writelist));
if (!wl) {
result = ERROR(memory_allocation);
result = MT_ERROR(memory_allocation);
goto error_unlock;
}
wl->out.buf = 0;
@@ -311,7 +320,7 @@ static void *pt_decompress(void *arg)
out = &wl->out;
/* zero should not happen here! */
result = pt_read(ctx, in, &wl->frame);
result = pt_read(ctx, in, &wl->frame, &wl->out.size);
if (BROTLIMT_isError(result)) {
list_move(&wl->node, &ctx->writelist_free);
goto error_lock;
@@ -320,43 +329,24 @@ static void *pt_decompress(void *arg)
if (in->size == 0)
break;
{
/* get frame size for output buffer */
unsigned char *src = (unsigned char *)in->buf + 6;
out->size = (size_t) MEM_readLE64(src);
}
if (out->allocated < out->size) {
if (out->allocated)
out->buf = realloc(out->buf, out->size);
else
out->buf = malloc(out->size);
if (!out->buf) {
result = ERROR(memory_allocation);
result = MT_ERROR(memory_allocation);
goto error_lock;
}
out->allocated = out->size;
}
/*
* size_t encoded_size,
* const uint8_t encoded_buffer[BROTLI_ARRAY_PARAM(encoded_size)],
* size_t* decoded_size,
* uint8_t decoded_buffer[BROTLI_ARRAY_PARAM(*decoded_size)]);
*/
result =
rv =
BrotliDecoderDecompress(in->size, in->buf, &out->size,
out->buf);
#if 0
if (BROF_isError(result)) {
bromt_errcode = result;
result = ERROR(compression_library);
goto error_lock;
}
#endif
if (result != 0) {
result = ERROR(frame_decompress);
if (rv != BROTLI_DECODER_RESULT_SUCCESS) {
result = MT_ERROR(frame_decompress);
goto error_lock;
}
@@ -386,113 +376,6 @@ static void *pt_decompress(void *arg)
return (void *)result;
}
/* single threaded */
static size_t st_decompress(void *arg)
{
BROTLIMT_DCtx *ctx = (BROTLIMT_DCtx *) arg;
BrotliDecoderResult nextToLoad = 0;
cwork_t *w = &ctx->cwork[0];
BROTLIMT_Buffer Out;
BROTLIMT_Buffer *out = &Out;
BROTLIMT_Buffer *in = &w->in;
void *magic = in->buf;
size_t pos = 0;
int rv;
/* allocate space for input buffer */
in->size = ctx->inputsize;
in->buf = malloc(in->size);
if (!in->buf)
return ERROR(memory_allocation);
/* allocate space for output buffer */
out->size = ctx->inputsize;
out->buf = malloc(out->size);
if (!out->buf) {
free(in->buf);
return ERROR(memory_allocation);
}
/* we have read already 4 bytes */
in->size = 4;
memcpy(in->buf, magic, in->size);
#if 0
nextToLoad =
BROF_decompress(w->dctx, out->buf, &pos, in->buf, &in->size, 0);
if (BROF_isError(nextToLoad)) {
free(in->buf);
free(out->buf);
return ERROR(compression_library);
}
#endif
for (; nextToLoad; pos = 0) {
if (nextToLoad > ctx->inputsize)
nextToLoad = ctx->inputsize;
/* read new input */
in->size = nextToLoad;
rv = ctx->fn_read(ctx->arg_read, in);
if (rv != 0) {
free(in->buf);
free(out->buf);
return mt_error(rv);
}
/* done, eof reached */
if (in->size == 0)
break;
/* still to read, or still to flush */
while ((pos < in->size) || (out->size == ctx->inputsize)) {
size_t remaining = in->size - pos;
out->size = ctx->inputsize;
/**
* BrotliDecoderResult BrotliDecoderDecompress(
* size_t encoded_size,
* const uint8_t encoded_buffer[BROTLI_ARRAY_PARAM(encoded_size)],
* size_t* decoded_size,
* uint8_t decoded_buffer[BROTLI_ARRAY_PARAM(*decoded_size)]);
*/
/* decompress */
#if 0
nextToLoad =
BrotliDecoderDecompress(out->buf, &out->size,
(unsigned char *)in->buf +
pos, &remaining, NULL);
if (BROF_isError(nextToLoad)) {
free(in->buf);
free(out->buf);
return ERROR(compression_library);
}
#endif
/* have some output */
if (out->size) {
rv = ctx->fn_write(ctx->arg_write, out);
if (rv != 0) {
free(in->buf);
free(out->buf);
return mt_error(rv);
}
}
if (nextToLoad == 0)
break;
pos += remaining;
}
}
/* no error */
free(out->buf);
free(in->buf);
return 0;
}
size_t BROTLIMT_decompressDCtx(BROTLIMT_DCtx * ctx, BROTLIMT_RdWr_t * rdwr)
{
unsigned char buf[4];
@@ -502,7 +385,7 @@ size_t BROTLIMT_decompressDCtx(BROTLIMT_DCtx * ctx, BROTLIMT_RdWr_t * rdwr)
void *retval_of_thread = 0;
if (!ctx)
return ERROR(compressionParameter_unsupported);
return MT_ERROR(compressionParameter_unsupported);
/* init reading and writing functions */
ctx->fn_read = rdwr->fn_read;
@@ -510,25 +393,18 @@ size_t BROTLIMT_decompressDCtx(BROTLIMT_DCtx * ctx, BROTLIMT_RdWr_t * rdwr)
ctx->arg_read = rdwr->arg_read;
ctx->arg_write = rdwr->arg_write;
/* check for BROFMT_MAGIC_SKIPPABLE */
/* check for BROTLIMT_MAGIC_SKIPPABLE */
in->buf = buf;
in->size = 4;
rv = ctx->fn_read(ctx->arg_read, in);
if (rv != 0)
return mt_error(rv);
if (in->size != 4)
return ERROR(data_error);
return MT_ERROR(data_error);
/* single threaded with unknown sizes */
if (MEM_readLE32(buf) != BROFMT_MAGIC_SKIPPABLE) {
/* look for correct magic */
if (MEM_readLE32(buf) != BROFMT_MAGICNUMBER)
return ERROR(data_error);
/* decompress single threaded */
return st_decompress(ctx);
}
if (MEM_readLE32(buf) != BROTLIMT_MAGIC_SKIPPABLE)
return MT_ERROR(data_error);
/* mark unused */
in->buf = 0;

10
C/zstdmt/lizard-mt_common.c
View File

@@ -9,7 +9,7 @@
* of patent rights can be found in the PATENTS file in the same directory.
*/
#include "lz5frame.h"
#include "lizframe.h"
#include "lizard-mt.h"
/* will be used for lib errors */
@@ -32,10 +32,10 @@ unsigned LIZARDMT_isError(size_t code)
*/
const char *LIZARDMT_getErrorString(size_t code)
{
if (LZ5F_isError(lizardmt_errcode))
return LZ5F_getErrorName(lizardmt_errcode);
if (LIZF_isError(lizardmt_errcode))
return LIZF_getErrorName(lizardmt_errcode);
static const char *notErrorCode = "Unspecified error lizardmt code";
static const char *noErrorCode = "Unspecified lizardmt error code";
switch ((LIZARDMT_ErrorCode) (0 - code)) {
case PREFIX(no_error):
return "No error detected";
@@ -57,6 +57,6 @@ const char *LIZARDMT_getErrorString(size_t code)
return "Compression library reports failure";
case PREFIX(maxCode):
default:
return notErrorCode;
return noErrorCode;
}
}

25
C/zstdmt/lizard-mt_compress.c
View File

@@ -14,8 +14,8 @@
#include <stdlib.h>
#include <string.h>
#define LZ5F_DISABLE_OBSOLETE_ENUMS
#include "lz5frame.h"
#define LIZF_DISABLE_OBSOLETE_ENUMS
#include "lizframe.h"
#include "memmt.h"
#include "threading.h"
@@ -38,7 +38,7 @@
/* worker for compression */
typedef struct {
LIZARDMT_CCtx *ctx;
LZ5F_preferences_t zpref;
LIZF_preferences_t zpref;
pthread_t pthread;
} cwork_t;
@@ -104,7 +104,7 @@ LIZARDMT_CCtx *LIZARDMT_createCCtx(int threads, int level, int inputsize)
return 0;
/* check level */
if (level < 1 || level > LIZARDMT_LEVEL_MAX)
if (level < LIZARDMT_LEVEL_MIN || level > LIZARDMT_LEVEL_MAX)
return 0;
/* calculate chunksize for one thread */
@@ -138,12 +138,12 @@ LIZARDMT_CCtx *LIZARDMT_createCCtx(int threads, int level, int inputsize)
w->ctx = ctx;
/* setup preferences for that thread */
memset(&w->zpref, 0, sizeof(LZ5F_preferences_t));
memset(&w->zpref, 0, sizeof(LIZF_preferences_t));
w->zpref.compressionLevel = level;
w->zpref.frameInfo.blockMode = LZ5F_blockLinked;
w->zpref.frameInfo.blockMode = LIZF_blockLinked;
w->zpref.frameInfo.contentSize = 1;
w->zpref.frameInfo.contentChecksumFlag =
LZ5F_contentChecksumEnabled;
LIZF_contentChecksumEnabled;
}
@@ -169,7 +169,6 @@ static size_t mt_error(int rv)
return ERROR(memory_allocation);
}
/* XXX, some catch all other errors */
return ERROR(read_fail);
}
@@ -230,7 +229,7 @@ static void *pt_compress(void *arg)
entry = list_first(&ctx->writelist_free);
wl = list_entry(entry, struct writelist, node);
wl->out.size =
LZ5F_compressFrameBound(ctx->inputsize,
LIZF_compressFrameBound(ctx->inputsize,
&w->zpref) + 12;
list_move(entry, &ctx->writelist_busy);
} else {
@@ -242,7 +241,7 @@ static void *pt_compress(void *arg)
return (void *)ERROR(memory_allocation);
}
wl->out.size =
LZ5F_compressFrameBound(ctx->inputsize,
LIZF_compressFrameBound(ctx->inputsize,
&w->zpref) + 12;;
wl->out.buf = malloc(wl->out.size);
if (!wl->out.buf) {
@@ -263,7 +262,7 @@ static void *pt_compress(void *arg)
}
/* eof */
if (in.size == 0) {
if (in.size == 0 && ctx->frames > 0) {
free(in.buf);
pthread_mutex_unlock(&ctx->read_mutex);
@@ -279,10 +278,10 @@ static void *pt_compress(void *arg)
/* compress whole frame */
result =
LZ5F_compressFrame((unsigned char *)wl->out.buf + 12,
LIZF_compressFrame((unsigned char *)wl->out.buf + 12,
wl->out.size - 12, in.buf, in.size,
&w->zpref);
if (LZ5F_isError(result)) {
if (LIZF_isError(result)) {
pthread_mutex_lock(&ctx->write_mutex);
list_move(&wl->node, &ctx->writelist_free);
pthread_mutex_unlock(&ctx->write_mutex);

30
C/zstdmt/lizard-mt_decompress.c
View File

@@ -14,8 +14,8 @@
#include <stdlib.h>
#include <string.h>
#define LZ5F_DISABLE_OBSOLETE_ENUMS
#include "lz5frame.h"
#define LIZF_DISABLE_OBSOLETE_ENUMS
#include "lizframe.h"
#include "memmt.h"
#include "threading.h"
@@ -40,7 +40,7 @@ typedef struct {
LIZARDMT_DCtx *ctx;
pthread_t pthread;
LIZARDMT_Buffer in;
LZ5F_decompressionContext_t dctx;
LIZF_decompressionContext_t dctx;
} cwork_t;
struct writelist;
@@ -130,7 +130,7 @@ LIZARDMT_DCtx *LIZARDMT_createDCtx(int threads, int inputsize)
w->ctx = ctx;
/* setup thread work */
LZ5F_createDecompressionContext(&w->dctx, LZ5F_VERSION);
LIZF_createDecompressionContext(&w->dctx, LIZF_VERSION);
}
return ctx;
@@ -325,12 +325,16 @@ static void *pt_decompress(void *arg)
if (in->size == 0)
break;
{
/* mininmal frame */
if (in->size < 40 && ctx->frames == 1) {
out->size = 1024 * 64;
} else {
/* get frame size for output buffer */
unsigned char *src = (unsigned char *)in->buf + 6;
out->size = (size_t) MEM_readLE64(src);
}
if (out->allocated < out->size) {
if (out->allocated)
out->buf = realloc(out->buf, out->size);
@@ -344,10 +348,10 @@ static void *pt_decompress(void *arg)
}
result =
LZ5F_decompress(w->dctx, out->buf, &out->size,
LIZF_decompress(w->dctx, out->buf, &out->size,
in->buf, &in->size, 0);
if (LZ5F_isError(result)) {
if (LIZF_isError(result)) {
lizardmt_errcode = result;
result = ERROR(compression_library);
goto error_lock;
@@ -388,7 +392,7 @@ static void *pt_decompress(void *arg)
static size_t st_decompress(void *arg)
{
LIZARDMT_DCtx *ctx = (LIZARDMT_DCtx *) arg;
LZ5F_errorCode_t nextToLoad = 0;
LIZF_errorCode_t nextToLoad = 0;
cwork_t *w = &ctx->cwork[0];
LIZARDMT_Buffer Out;
LIZARDMT_Buffer *out = &Out;
@@ -416,8 +420,8 @@ static size_t st_decompress(void *arg)
memcpy(in->buf, magic, in->size);
nextToLoad =
LZ5F_decompress(w->dctx, out->buf, &pos, in->buf, &in->size, 0);
if (LZ5F_isError(nextToLoad)) {
LIZF_decompress(w->dctx, out->buf, &pos, in->buf, &in->size, 0);
if (LIZF_isError(nextToLoad)) {
free(in->buf);
free(out->buf);
return ERROR(compression_library);
@@ -447,10 +451,10 @@ static size_t st_decompress(void *arg)
/* decompress */
nextToLoad =
LZ5F_decompress(w->dctx, out->buf, &out->size,
LIZF_decompress(w->dctx, out->buf, &out->size,
(unsigned char *)in->buf + pos,
&remaining, NULL);
if (LZ5F_isError(nextToLoad)) {
if (LIZF_isError(nextToLoad)) {
free(in->buf);
free(out->buf);
return ERROR(compression_library);
@@ -599,7 +603,7 @@ void LIZARDMT_freeDCtx(LIZARDMT_DCtx * ctx)
for (t = 0; t < ctx->threads; t++) {
cwork_t *w = &ctx->cwork[t];
LZ5F_freeDecompressionContext(w->dctx);
LIZF_freeDecompressionContext(w->dctx);
}
pthread_mutex_destroy(&ctx->read_mutex);

1
C/zstdmt/lz4-mt.h
View File

@@ -26,6 +26,7 @@ extern "C" {
/* current maximum the library will accept */
#define LZ4MT_THREAD_MAX 128
#define LZ4MT_LEVEL_MIN 1
#define LZ4MT_LEVEL_MAX 12
#define LZ4FMT_MAGICNUMBER 0x184D2204U

4
C/zstdmt/lz4-mt_common.c
View File

@@ -35,7 +35,7 @@ const char *LZ4MT_getErrorString(size_t code)
if (LZ4F_isError(lz4mt_errcode))
return LZ4F_getErrorName(lz4mt_errcode);
static const char *notErrorCode = "Unspecified error lz4mt code";
static const char *noErrorCode = "Unspecified lz4mt error code";
switch ((LZ4MT_ErrorCode) (0 - code)) {
case PREFIX(no_error):
return "No error detected";
@@ -57,6 +57,6 @@ const char *LZ4MT_getErrorString(size_t code)
return "Compression library reports failure";
case PREFIX(maxCode):
default:
return notErrorCode;
return noErrorCode;
}
}

7
C/zstdmt/lz4-mt_compress.c
View File

@@ -104,7 +104,7 @@ LZ4MT_CCtx *LZ4MT_createCCtx(int threads, int level, int inputsize)
return 0;
/* check level */
if (level < 1 || level > LZ4MT_LEVEL_MAX)
if (level < LZ4MT_LEVEL_MIN || level > LZ4MT_LEVEL_MAX)
return 0;
/* calculate chunksize for one thread */
@@ -169,7 +169,6 @@ static size_t mt_error(int rv)
return ERROR(memory_allocation);
}
/* XXX, some catch all other errors */
return ERROR(read_fail);
}
@@ -261,9 +260,9 @@ static void *pt_compress(void *arg)
pthread_mutex_unlock(&ctx->read_mutex);
return (void *)mt_error(rv);
}
/* eof */
if (in.size == 0) {
if (in.size == 0 && ctx->frames > 0) {
free(in.buf);
pthread_mutex_unlock(&ctx->read_mutex);

5
C/zstdmt/lz4-mt_decompress.c
View File

@@ -325,7 +325,10 @@ static void *pt_decompress(void *arg)
if (in->size == 0)
break;
{
/* mininmal frame */
if (in->size < 40 && ctx->frames == 1) {
out->size = 1024 * 64;
} else {
/* get frame size for output buffer */
unsigned char *src = (unsigned char *)in->buf + 6;
out->size = (size_t) MEM_readLE64(src);

1
C/zstdmt/lz5-mt.h
View File

@@ -26,6 +26,7 @@ extern "C" {
/* current maximum the library will accept */
#define LZ5MT_THREAD_MAX 128
#define LZ5MT_LEVEL_MIN 1
#define LZ5MT_LEVEL_MAX 15
#define LZ5FMT_MAGICNUMBER 0x184D2205U

4
C/zstdmt/lz5-mt_common.c
View File

@@ -35,7 +35,7 @@ const char *LZ5MT_getErrorString(size_t code)
if (LZ5F_isError(lz5mt_errcode))
return LZ5F_getErrorName(lz5mt_errcode);
static const char *notErrorCode = "Unspecified error lz5mt code";
static const char *noErrorCode = "Unspecified lz5mt error code";
switch ((LZ5MT_ErrorCode) (0 - code)) {
case PREFIX(no_error):
return "No error detected";
@@ -57,6 +57,6 @@ const char *LZ5MT_getErrorString(size_t code)
return "Compression library reports failure";
case PREFIX(maxCode):
default:
return notErrorCode;
return noErrorCode;
}
}

5
C/zstdmt/lz5-mt_compress.c
View File

@@ -104,7 +104,7 @@ LZ5MT_CCtx *LZ5MT_createCCtx(int threads, int level, int inputsize)
return 0;
/* check level */
if (level < 1 || level > LZ5MT_LEVEL_MAX)
if (level < LZ5MT_LEVEL_MIN || level > LZ5MT_LEVEL_MAX)
return 0;
/* calculate chunksize for one thread */
@@ -169,7 +169,6 @@ static size_t mt_error(int rv)
return ERROR(memory_allocation);
}
/* XXX, some catch all other errors */
return ERROR(read_fail);
}
@@ -263,7 +262,7 @@ static void *pt_compress(void *arg)
}
/* eof */
if (in.size == 0) {
if (in.size == 0 && ctx->frames > 0) {
free(in.buf);
pthread_mutex_unlock(&ctx->read_mutex);

5
C/zstdmt/lz5-mt_decompress.c
View File

@@ -325,7 +325,10 @@ static void *pt_decompress(void *arg)
if (in->size == 0)
break;
{
/* mininmal frame */
if (in->size < 40 && ctx->frames == 1) {
out->size = 1024 * 64;
} else {
/* get frame size for output buffer */
unsigned char *src = (unsigned char *)in->buf + 6;
out->size = (size_t) MEM_readLE64(src);

1
C/zstdmt/zstd-mt.h
View File

@@ -25,6 +25,7 @@ extern "C" {
#include <stddef.h> /* size_t */
#define ZSTDMT_THREAD_MAX 128
#define ZSTDMT_LEVEL_MIN 1
#define ZSTDMT_LEVEL_MAX 22
/* zstd magic values */

4
C/zstdmt/zstd-mt_common.c
View File

@@ -31,7 +31,7 @@ unsigned ZSTDMT_isError(size_t code)
*/
const char *ZSTDMT_getErrorString(size_t code)
{
static const char *notErrorCode = "Unspecified error zstmt code";
static const char *noErrorCode = "Unspecified zstmt error code";
if (ZSTD_isError(zstdmt_errcode))
return ZSTD_getErrorName(zstdmt_errcode);
@@ -57,6 +57,6 @@ const char *ZSTDMT_getErrorString(size_t code)
return "Compression library reports failure";
case ZSTDMT_PREFIX(maxCode):
default:
return notErrorCode;
return noErrorCode;
}
}

6
C/zstdmt/zstd-mt_compress.c
View File

@@ -79,7 +79,6 @@ struct ZSTDMT_CCtx_s {
/* error handling */
pthread_mutex_t error_mutex;
size_t zstd_errcode;
size_t zstdmt_errcode;
/* lists for writing queue */
@@ -107,7 +106,7 @@ ZSTDMT_CCtx *ZSTDMT_createCCtx(int threads, int level, int inputsize)
goto err_ctx;
/* check level */
if (level < 1 || level > ZSTDMT_LEVEL_MAX)
if (level < ZSTDMT_LEVEL_MIN || level > ZSTDMT_LEVEL_MAX)
goto err_ctx;
/* calculate chunksize for one thread */
@@ -173,7 +172,6 @@ static size_t mt_error(int rv)
return ZSTDMT_ERROR(memory_allocation);
}
/* XXX, some catch all other errors */
return ZSTDMT_ERROR(read_fail);
}
@@ -270,7 +268,7 @@ static void *pt_compress(void *arg)
}
/* eof */
if (in.size == 0) {
if (in.size == 0 && ctx->frames > 0) {
free(in.buf);
pthread_mutex_unlock(&ctx->read_mutex);