xevion/easy7zip
1
0
Fork 0
mirror of https://github.com/Xevion/easy7zip.git synced 2025-12-06 11:14:58 -06:00
Code Issues Packages Projects Releases Wiki Activity

Remove last zstd objects of my zstdmt library

Browse Source
This commit is contained in:
Tino Reichardt
2018-10-29 18:44:53 +01:00
parent 54a9234630
commit 4a4c2e307d
7 changed files with 0 additions and 1606 deletions
Download Patch File Download Diff File Expand all files Collapse all files

210
C/zstdmt/zstd-mt.h
View File

@@ -1,210 +0,0 @@
/**
* Copyright (c) 2016 Tino Reichardt
* All rights reserved.
*
* This source code is licensed under the BSD-style license found in the
* LICENSE file in the root directory of this source tree. An additional grant
* of patent rights can be found in the PATENTS file in the same directory.
*
* You can contact the author at:
* - zstdmt source repository: https://github.com/mcmilk/zstdmt
*/
/* ***************************************
* Defines
****************************************/
#ifndef ZSTDCB_H
#define ZSTDCB_H
#if defined (__cplusplus)
extern "C" {
#endif
#include <stddef.h> /* size_t */
#define ZSTDCB_THREAD_MAX 128
#define ZSTDCB_LEVEL_MIN 1
#define ZSTDCB_LEVEL_MAX 22
/* zstd magic values */
#define ZSTDCB_MAGICNUMBER_V01 0x1EB52FFDU
#define ZSTDCB_MAGICNUMBER_MIN 0xFD2FB522U
#define ZSTDCB_MAGICNUMBER_MAX 0xFD2FB528U
#define ZSTDCB_MAGIC_SKIPPABLE 0x184D2A50U
/* **************************************
* Error Handling
****************************************/
typedef enum {
ZSTDCB_error_no_error,
ZSTDCB_error_memory_allocation,
ZSTDCB_error_init_missing,
ZSTDCB_error_read_fail,
ZSTDCB_error_write_fail,
ZSTDCB_error_data_error,
ZSTDCB_error_frame_compress,
ZSTDCB_error_frame_decompress,
ZSTDCB_error_compressionParameter_unsupported,
ZSTDCB_error_compression_library,
ZSTDCB_error_canceled,
ZSTDCB_error_maxCode
} ZSTDCB_ErrorCode;
extern size_t zstdmt_errcode;
#define ZSTDCB_PREFIX(name) ZSTDCB_error_##name
#define ZSTDCB_ERROR(name) ((size_t)-ZSTDCB_PREFIX(name))
extern unsigned ZSTDCB_isError(size_t code);
extern const char* ZSTDCB_getErrorString(size_t code);
/* **************************************
* Structures
****************************************/
typedef struct {
void *buf; /* ptr to data */
size_t size; /* current filled in buf */
size_t allocated; /* length of buf */
} ZSTDCB_Buffer;
/**
* reading and writing functions
* - you can use stdio functions or plain read/write
* - just write some wrapper on your own
* - a sample is given in 7-Zip ZS
*
* error definitions:
* 0 = success
* -1 = generic read/write error
* -2 = user abort
* -3 = memory
*/
typedef int (fn_read) (void *args, ZSTDCB_Buffer * in);
typedef int (fn_write) (void *args, ZSTDCB_Buffer * out);
typedef struct {
fn_read *fn_read;
void *arg_read;
fn_write *fn_write;
void *arg_write;
} ZSTDCB_RdWr_t;
/* **************************************
* Compression
****************************************/
typedef struct ZSTDCB_CCtx_s ZSTDCB_CCtx;
/**
* ZSTDCB_createCCtx() - allocate new compression context
*
* This function allocates and initializes an zstd commpression context.
* The context can be used multiple times without the need for resetting
* or re-initializing.
*
* @level: compression level, which should be used (1..22)
* @threads: number of threads, which should be used (1..ZSTDCB_THREAD_MAX)
* @inputsize: - if zero, becomes some optimal value for the level
* - if nonzero, the given value is taken
* @zstdmt_errcode: space for storing zstd errors (needed for thread safety)
* @return: the context on success, zero on error
*/
ZSTDCB_CCtx *ZSTDCB_createCCtx(int threads, int level, int inputsize);
/**
* ZSTDCB_compressDCtx() - threaded compression for zstd
*
* This function will create valid zstd streams. The number of threads,
* the input chunksize and the compression level are ....
*
* @ctx: context, which needs to be created with ZSTDCB_createDCtx()
* @rdwr: callback structure, which defines reding/writing functions
* @return: zero on success, or error code
*/
size_t ZSTDCB_compressCCtx(ZSTDCB_CCtx * ctx, ZSTDCB_RdWr_t * rdwr);
/**
* ZSTDCB_GetFramesCCtx() - number of written frames
* ZSTDCB_GetInsizeCCtx() - read bytes of input
* ZSTDCB_GetOutsizeCCtx() - written bytes of output
*
* These three functions will return some statistical data of the
* compression context ctx.
*
* @ctx: context, which should be examined
* @return: the request value, or zero on error
*/
size_t ZSTDCB_GetFramesCCtx(ZSTDCB_CCtx * ctx);
size_t ZSTDCB_GetInsizeCCtx(ZSTDCB_CCtx * ctx);
size_t ZSTDCB_GetOutsizeCCtx(ZSTDCB_CCtx * ctx);
/**
* ZSTDCB_freeCCtx() - free compression context
*
* This function will free all allocated resources, which were allocated
* by ZSTDCB_createCCtx(). This function can not fail.
*
* @ctx: context, which should be freed
*/
void ZSTDCB_freeCCtx(ZSTDCB_CCtx * ctx);
/* **************************************
* Decompression
****************************************/
typedef struct ZSTDCB_DCtx_s ZSTDCB_DCtx;
/**
* 1) allocate new cctx
* - return cctx or zero on error
*
* @level - 1 .. 22
* @threads - 1 .. ZSTDCB_THREAD_MAX
* @srclen - the max size of src for ZSTDCB_compressCCtx()
* @dstlen - the min size of dst
*/
ZSTDCB_DCtx *ZSTDCB_createDCtx(int threads, int inputsize);
/**
* ZSTDCB_decompressDCtx() - threaded decompression for zstd
*
* This function will decompress valid zstd streams.
*
* @ctx: context, which needs to be created with ZSTDCB_createDCtx()
* @rdwr: callback structure, which defines reding/writing functions
* @return: zero on success, or error code
*/
size_t ZSTDCB_decompressDCtx(ZSTDCB_DCtx * ctx, ZSTDCB_RdWr_t * rdwr);
/**
* ZSTDCB_GetFramesDCtx() - number of read frames
* ZSTDCB_GetInsizeDCtx() - read bytes of input
* ZSTDCB_GetOutsizeDCtx() - written bytes of output
*
* These three functions will return some statistical data of the
* decompression context ctx.
*
* @ctx: context, which should be examined
* @return: the request value, or zero on error
*/
size_t ZSTDCB_GetFramesDCtx(ZSTDCB_DCtx * ctx);
size_t ZSTDCB_GetInsizeDCtx(ZSTDCB_DCtx * ctx);
size_t ZSTDCB_GetOutsizeDCtx(ZSTDCB_DCtx * ctx);
/**
* ZSTDCB_freeDCtx() - free decompression context
*
* This function will free all allocated resources, which were allocated
* by ZSTDCB_createDCtx(). This function can not fail.
*
* @ctx: context, which should be freed
*/
void ZSTDCB_freeDCtx(ZSTDCB_DCtx * ctx);
#if defined (__cplusplus)
}
#endif
#endif /* ZSTDCB_H */

62
C/zstdmt/zstd-mt_common.c
View File

@@ -1,62 +0,0 @@
/**
* Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
* Copyright (c) 2016 Tino Reichardt
* All rights reserved.
*
* This source code is licensed under the BSD-style license found in the
* LICENSE file in the root directory of this source tree. An additional grant
* of patent rights can be found in the PATENTS file in the same directory.
*/
#include "zstd.h"
#include "zstd-mt.h"
/* ****************************************
* ZSTDMT Error Management
******************************************/
size_t zstdmt_errcode;
/**
* ZSTDCB_isError() - tells if a return value is an error code
*/
unsigned ZSTDCB_isError(size_t code)
{
return (code > ZSTDCB_ERROR(maxCode));
}
/**
* LZ4MT_getErrorString() - give error code string from function result
*/
const char *ZSTDCB_getErrorString(size_t code)
{
static const char *noErrorCode = "Unspecified zstmt error code";
if (ZSTD_isError(zstdmt_errcode))
return ZSTD_getErrorName(zstdmt_errcode);
switch ((ZSTDCB_ErrorCode) (0 - code)) {
case ZSTDCB_PREFIX(no_error):
return "No error detected";
case ZSTDCB_PREFIX(memory_allocation):
return "Allocation error : not enough memory";
case ZSTDCB_PREFIX(read_fail):
return "Read failure";
case ZSTDCB_PREFIX(write_fail):
return "Write failure";
case ZSTDCB_PREFIX(data_error):
return "Malformed input";
case ZSTDCB_PREFIX(frame_compress):
return "Could not compress frame at once";
case ZSTDCB_PREFIX(frame_decompress):
return "Could not decompress frame at once";
case ZSTDCB_PREFIX(compressionParameter_unsupported):
return "Compression parameter is out of bound";
case ZSTDCB_PREFIX(compression_library):
return "Compression library reports failure";
case ZSTDCB_PREFIX(maxCode):
default:
return noErrorCode;
}
}

435
C/zstdmt/zstd-mt_compress.c
View File

@@ -1,435 +0,0 @@
/**
* Copyright (c) 2016 - 2017 Tino Reichardt
* All rights reserved.
*
* This source code is licensed under the BSD-style license found in the
* LICENSE file in the root directory of this source tree. An additional grant
* of patent rights can be found in the PATENTS file in the same directory.
*
* You can contact the author at:
* - zstdmt source repository: https://github.com/mcmilk/zstdmt
*/
#include <stdio.h>
#include <stdlib.h>
#define ZSTD_STATIC_LINKING_ONLY
#include "zstd.h"
#include "memmt.h"
#include "threading.h"
#include "list.h"
#include "zstd-mt.h"
/**
* multi threaded zstd compression
*
* - each thread works on his own
* - needs a callback for reading / writing
* - each worker does this:
* 1) get read mutex and read some input
* 2) release read mutex and do compression
* 3) get write mutex and write result
* 4) begin with step 1 again, until no input
*/
/* worker for compression */
typedef struct {
ZSTDCB_CCtx *ctx;
pthread_t pthread;
} cwork_t;
struct writelist;
struct writelist {
size_t frame;
ZSTDCB_Buffer out;
struct list_head node;
};
struct ZSTDCB_CCtx_s {
/* level: 1..ZSTDCB_LEVEL_MAX */
int level;
/* threads: 1..ZSTDCB_THREAD_MAX */
int threads;
/* buffersize for reading input */
int inputsize;
/* statistic */
size_t insize;
size_t outsize;
size_t curframe;
size_t frames;
/* threading */
cwork_t *cwork;
/* reading input */
pthread_mutex_t read_mutex;
fn_read *fn_read;
void *arg_read;
/* writing output */
pthread_mutex_t write_mutex;
fn_write *fn_write;
void *arg_write;
/* error handling */
pthread_mutex_t error_mutex;
size_t zstdmt_errcode;
/* lists for writing queue */
struct list_head writelist_free;
struct list_head writelist_busy;
struct list_head writelist_done;
};
/* **************************************
* Compression
****************************************/
ZSTDCB_CCtx *ZSTDCB_createCCtx(int threads, int level, int inputsize)
{
ZSTDCB_CCtx *ctx;
int t;
/* allocate ctx */
ctx = (ZSTDCB_CCtx *) malloc(sizeof(ZSTDCB_CCtx));
if (!ctx)
return 0;
/* check threads value */
if (threads < 1 || threads > ZSTDCB_THREAD_MAX)
goto err_ctx;
/* check level */
if (level < ZSTDCB_LEVEL_MIN || level > ZSTDCB_LEVEL_MAX)
goto err_ctx;
/* calculate chunksize for one thread */
if (inputsize)
ctx->inputsize = inputsize;
else {
const int windowLog[] = {
19, 19, 20, 20, 20, /* 1 - 5 */
21, 21, 21, 21, 21, /* 6 - 10 */
22, 22, 22, 22, 22, /* 11 - 15 */
23, 23, 23, 23, 25, /* 16 - 20 */
26, 27
};
ctx->inputsize = 1 << (windowLog[level - 1] + 1);
}
/* setup ctx */
ctx->level = level;
ctx->threads = threads;
pthread_mutex_init(&ctx->read_mutex, NULL);
pthread_mutex_init(&ctx->write_mutex, NULL);
pthread_mutex_init(&ctx->error_mutex, NULL);
INIT_LIST_HEAD(&ctx->writelist_free);
INIT_LIST_HEAD(&ctx->writelist_busy);
INIT_LIST_HEAD(&ctx->writelist_done);
ctx->cwork = (cwork_t *) malloc(sizeof(cwork_t) * threads);
if (!ctx->cwork)
goto err_ctx;
for (t = 0; t < ctx->threads; t++) {
cwork_t *w = &ctx->cwork[t];
w->ctx = ctx;
}
return ctx;
err_ctx:
free(ctx);
return 0;
}
/**
* mt_error - return mt lib specific error code
*/
static size_t mt_error(int rv)
{
switch (rv) {
case -1:
return ZSTDCB_ERROR(read_fail);
case -2:
return ZSTDCB_ERROR(canceled);
case -3:
return ZSTDCB_ERROR(memory_allocation);
}
return ZSTDCB_ERROR(read_fail);
}
/**
* pt_write - queue for compressed output
*/
static size_t pt_write(ZSTDCB_CCtx * ctx, struct writelist *wl)
{
struct list_head *entry;
int rv;
/* move the entry to the done list */
list_move(&wl->node, &ctx->writelist_done);
/* the entry isn't the currently needed, return... */
if (wl->frame != ctx->curframe)
return 0;
again:
/* check, what can be written ... */
list_for_each(entry, &ctx->writelist_done) {
wl = list_entry(entry, struct writelist, node);
if (wl->frame == ctx->curframe) {
rv = ctx->fn_write(ctx->arg_write, &wl->out);
if (rv != 0)
return mt_error(rv);
ctx->outsize += wl->out.size;
ctx->curframe++;
list_move(entry, &ctx->writelist_free);
goto again;
}
}
return 0;
}
/* parallel compression worker */
static void *pt_compress(void *arg)
{
cwork_t *w = (cwork_t *) arg;
ZSTDCB_CCtx *ctx = w->ctx;
struct writelist *wl;
size_t result;
ZSTDCB_Buffer in;
/* inbuf is constant */
in.size = ctx->inputsize;
in.buf = malloc(in.size);
if (!in.buf)
return (void *)ZSTDCB_ERROR(memory_allocation);
for (;;) {
struct list_head *entry;
ZSTDCB_Buffer *out;
int rv;
/* allocate space for new output */
pthread_mutex_lock(&ctx->write_mutex);
if (!list_empty(&ctx->writelist_free)) {
/* take unused entry */
entry = list_first(&ctx->writelist_free);
wl = list_entry(entry, struct writelist, node);
wl->out.size = ZSTD_compressBound(ctx->inputsize) + 12;
list_move(entry, &ctx->writelist_busy);
} else {
/* allocate new one */
wl = (struct writelist *)
malloc(sizeof(struct writelist));
if (!wl) {
pthread_mutex_unlock(&ctx->write_mutex);
free(in.buf);
return (void *)ZSTDCB_ERROR(memory_allocation);
}
wl->out.size = ZSTD_compressBound(ctx->inputsize) + 12;;
wl->out.buf = malloc(wl->out.size);
if (!wl->out.buf) {
pthread_mutex_unlock(&ctx->write_mutex);
free(in.buf);
return (void *)ZSTDCB_ERROR(memory_allocation);
}
list_add(&wl->node, &ctx->writelist_busy);
}
pthread_mutex_unlock(&ctx->write_mutex);
out = &wl->out;
/* read new input */
pthread_mutex_lock(&ctx->read_mutex);
in.size = ctx->inputsize;
rv = ctx->fn_read(ctx->arg_read, &in);
if (rv != 0) {
pthread_mutex_unlock(&ctx->read_mutex);
result = mt_error(rv);
goto error;
}
/* eof */
if (in.size == 0 && ctx->frames > 0) {
free(in.buf);
pthread_mutex_unlock(&ctx->read_mutex);
pthread_mutex_lock(&ctx->write_mutex);
list_move(&wl->node, &ctx->writelist_free);
pthread_mutex_unlock(&ctx->write_mutex);
goto okay;
}
ctx->insize += in.size;
wl->frame = ctx->frames++;
pthread_mutex_unlock(&ctx->read_mutex);
/* compress whole frame */
{
unsigned char *outbuf = out->buf;
result =
ZSTD_compress(outbuf + 12, out->size - 12, in.buf,
in.size, ctx->level);
if (ZSTD_isError(result)) {
zstdmt_errcode = result;
result = ZSTDCB_ERROR(compression_library);
goto error;
}
}
/* write skippable frame */
{
unsigned char *outbuf = out->buf;
MEM_writeLE32(outbuf + 0, ZSTDCB_MAGIC_SKIPPABLE);
MEM_writeLE32(outbuf + 4, 4);
MEM_writeLE32(outbuf + 8, (U32) result);
out->size = result + 12;
}
/* write result */
pthread_mutex_lock(&ctx->write_mutex);
result = pt_write(ctx, wl);
pthread_mutex_unlock(&ctx->write_mutex);
if (ZSTDCB_isError(result))
goto error;
}
okay:
return 0;
error:
pthread_mutex_lock(&ctx->write_mutex);
list_move(&wl->node, &ctx->writelist_free);
pthread_mutex_unlock(&ctx->write_mutex);
return (void *)result;
}
/* compress data, until input ends */
size_t ZSTDCB_compressCCtx(ZSTDCB_CCtx * ctx, ZSTDCB_RdWr_t * rdwr)
{
int t;
void *retval_of_thread = 0;
if (!ctx)
return ZSTDCB_ERROR(init_missing);
/* setup reading and writing functions */
ctx->fn_read = rdwr->fn_read;
ctx->fn_write = rdwr->fn_write;
ctx->arg_read = rdwr->arg_read;
ctx->arg_write = rdwr->arg_write;
/* init counter and error codes */
ctx->insize = 0;
ctx->outsize = 0;
ctx->frames = 0;
ctx->curframe = 0;
ctx->zstdmt_errcode = 0;
/* start all workers */
for (t = 0; t < ctx->threads; t++) {
cwork_t *w = &ctx->cwork[t];
pthread_create(&w->pthread, NULL, pt_compress, w);
}
/* wait for all workers */
for (t = 0; t < ctx->threads; t++) {
cwork_t *w = &ctx->cwork[t];
void *p = 0;
pthread_join(w->pthread, &p);
if (p)
retval_of_thread = p;
}
/* clean up the free list */
while (!list_empty(&ctx->writelist_free)) {
struct writelist *wl;
struct list_head *entry;
entry = list_first(&ctx->writelist_free);
wl = list_entry(entry, struct writelist, node);
free(wl->out.buf);
list_del(&wl->node);
free(wl);
}
/* on error, these two lists may have some entries */
if (retval_of_thread) {
struct writelist *wl;
struct list_head *entry;
while (!list_empty(&ctx->writelist_busy)) {
entry = list_first(&ctx->writelist_busy);
wl = list_entry(entry, struct writelist, node);
free(wl->out.buf);
list_del(&wl->node);
free(wl);
}
while (!list_empty(&ctx->writelist_done)) {
entry = list_first(&ctx->writelist_done);
wl = list_entry(entry, struct writelist, node);
free(wl->out.buf);
list_del(&wl->node);
free(wl);
}
}
return (size_t) retval_of_thread;
}
/* returns current uncompressed data size */
size_t ZSTDCB_GetInsizeCCtx(ZSTDCB_CCtx * ctx)
{
if (!ctx)
return ZSTDCB_ERROR(init_missing);
/* no mutex needed here */
return ctx->insize;
}
/* returns the current compressed data size */
size_t ZSTDCB_GetOutsizeCCtx(ZSTDCB_CCtx * ctx)
{
if (!ctx)
return ZSTDCB_ERROR(init_missing);
/* no mutex needed here */
return ctx->outsize;
}
/* returns the current compressed data frame count */
size_t ZSTDCB_GetFramesCCtx(ZSTDCB_CCtx * ctx)
{
if (!ctx)
return ZSTDCB_ERROR(init_missing);
/* no mutex needed here */
return ctx->curframe;
}
/* free all allocated buffers and structures */
void ZSTDCB_freeCCtx(ZSTDCB_CCtx * ctx)
{
if (!ctx)
return;
free(ctx->cwork);
free(ctx);
ctx = 0;
return;
}

891
C/zstdmt/zstd-mt_decompress.c
View File

@@ -1,891 +0,0 @@
/**
* Copyright (c) 2016 - 2017 Tino Reichardt
* All rights reserved.
*
* This source code is licensed under the BSD-style license found in the
* LICENSE file in the root directory of this source tree. An additional grant
* of patent rights can be found in the PATENTS file in the same directory.
*
* You can contact the author at:
* - zstdmt source repository: https://github.com/mcmilk/zstdmt
*/
#include <stdlib.h>
#include <string.h>
#define ZSTD_STATIC_LINKING_ONLY
#include "zstd.h"
#include "memmt.h"
#include "threading.h"
#include "list.h"
#include "zstd-mt.h"
/**
* multi threaded zstd decompression
*
* - each thread works on his own
* - needs a callback for reading / writing
* - each worker does this:
* 1) get read mutex and read some input
* 2) release read mutex and do decompression
* 3) get write mutex and write result
* 4) begin with step 1 again, until no input
*/
#if 0
#include <stdio.h>
#define dprintf(fmt, arg...) do { printf(fmt, ## arg); } while (0)
#else
#define dprintf(fmt, ...)
#endif /* DEBUG */
extern size_t zstdmt_errcode;
/* worker for compression */
typedef struct {
ZSTDCB_DCtx *ctx;
pthread_t pthread;
ZSTDCB_Buffer in;
ZSTD_DStream *dctx;
} cwork_t;
struct writelist;
struct writelist {
size_t frame;
ZSTDCB_Buffer out;
struct list_head node;
};
struct ZSTDCB_DCtx_s {
/* threads: 1..ZSTDCB_THREAD_MAX */
int threads;
int threadswanted;
/* input buffer, used at single threading */
size_t inputsize;
/* buffersize used for output */
size_t outputsize;
/* statistic */
size_t insize;
size_t outsize;
size_t curframe;
size_t frames;
/* threading */
cwork_t *cwork;
/* reading input */
pthread_mutex_t read_mutex;
fn_read *fn_read;
void *arg_read;
/* writing output */
pthread_mutex_t write_mutex;
fn_write *fn_write;
void *arg_write;
/* error handling */
pthread_mutex_t error_mutex;
/* lists for writing queue */
struct list_head writelist_free;
struct list_head writelist_busy;
struct list_head writelist_done;
};
/* **************************************
* Decompression
****************************************/
ZSTDCB_DCtx *ZSTDCB_createDCtx(int threads, int inputsize)
{
ZSTDCB_DCtx *ctx;
/* allocate ctx */
ctx = (ZSTDCB_DCtx *) malloc(sizeof(ZSTDCB_DCtx));
if (!ctx)
return 0;
/* check threads value */
if (threads < 1 || threads > ZSTDCB_THREAD_MAX)
return 0;
/* setup ctx */
ctx->threadswanted = threads;
ctx->threads = 0;
ctx->insize = 0;
ctx->outsize = 0;
ctx->frames = 0;
ctx->curframe = 0;
/* will be used for single stream only */
if (inputsize)
ctx->inputsize = inputsize;
else
ctx->inputsize = 1024 * 512;
/* frame size (will get higher, when needed) */
ctx->outputsize = 1024 * 512;
/* later */
ctx->cwork = 0;
return ctx;
}
/**
* IsZstd_Magic - check, if 4 bytes are valid ZSTD MAGIC
*/
static int IsZstd_Magic(unsigned char *buf)
{
U32 magic = MEM_readLE32(buf);
if (magic == ZSTDCB_MAGICNUMBER_V01)
return 1;
return (magic >= ZSTDCB_MAGICNUMBER_MIN
&& magic <= ZSTDCB_MAGICNUMBER_MAX);
}
/**
* IsZstd_Skippable - check, if 4 bytes are MAGIC_SKIPPABLE
*/
static int IsZstd_Skippable(unsigned char *buf)
{
return (MEM_readLE32(buf) == ZSTDCB_MAGIC_SKIPPABLE);
}
/**
* mt_error - return mt lib specific error code
*/
static size_t mt_error(int rv)
{
switch (rv) {
case -1:
return ZSTDCB_ERROR(read_fail);
case -2:
return ZSTDCB_ERROR(canceled);
case -3:
return ZSTDCB_ERROR(memory_allocation);
}
/* XXX, some catch all other errors */
return ZSTDCB_ERROR(read_fail);
}
/**
* pt_write - queue for decompressed output
*/
static size_t pt_write(ZSTDCB_DCtx * ctx, struct writelist *wl)
{
struct list_head *entry;
/* move the entry to the done list */
list_move(&wl->node, &ctx->writelist_done);
again:
/* check, what can be written ... */
list_for_each(entry, &ctx->writelist_done) {
wl = list_entry(entry, struct writelist, node);
if (wl->frame == ctx->curframe) {
int rv = ctx->fn_write(ctx->arg_write, &wl->out);
if (rv != 0)
return mt_error(rv);
ctx->outsize += wl->out.size;
ctx->curframe++;
list_move(entry, &ctx->writelist_free);
goto again;
}
}
return 0;
}
/**
* pt_read - read compressed input
*/
static size_t pt_read(ZSTDCB_DCtx * ctx, ZSTDCB_Buffer * in, size_t * frame)
{
unsigned char hdrbuf[12];
ZSTDCB_Buffer hdr;
size_t toRead;
int rv;
pthread_mutex_lock(&ctx->read_mutex);
/* special case, some bytes were read by magic check */
if (unlikely(ctx->frames == 0)) {
/* the magic check reads exactly 16 bytes! */
if (unlikely(in->size != 16))
goto error_data;
ctx->insize += 16;
/**
* zstdmt mode, with zstd magic prefix
* 9 bytes zero byte frame + 12 byte skippable
* - 21 bytes to read, 16 bytes done
* - read 5 bytes, put them together (12 byte hdr)
*/
if (!IsZstd_Skippable(in->buf)) {
memcpy(hdrbuf, in->buf, 7);
hdr.buf = hdrbuf + 7;
hdr.size = 5;
rv = ctx->fn_read(ctx->arg_read, &hdr);
if (rv != 0) {
pthread_mutex_unlock(&ctx->read_mutex);
return mt_error(rv);
}
if (hdr.size != 5)
goto error_data;
hdr.buf = hdrbuf;
ctx->insize += 16 + 5;
/* read data */
toRead = MEM_readLE32((unsigned char *)hdr.buf + 8);
in->size = toRead;
in->buf = malloc(in->size);
if (!in->buf)
goto error_nomem;
in->allocated = in->size;
rv = ctx->fn_read(ctx->arg_read, in);
if (rv != 0) {
pthread_mutex_unlock(&ctx->read_mutex);
return mt_error(rv);
}
if (in->size != toRead)
goto error_data;
ctx->insize += in->size;
*frame = ctx->frames++;
pthread_mutex_unlock(&ctx->read_mutex);
return 0; /* done! */
}
/**
* pzstd mode, no prefix
* - start directly with 12 byte skippable frame
*/
if (IsZstd_Skippable(in->buf)) {
unsigned char *start = in->buf; /* 16 bytes data */
toRead = MEM_readLE32((unsigned char *)start + 8);
in->size = toRead;
in->buf = malloc(in->size);
if (!in->buf)
goto error_nomem;
in->allocated = in->size;
/* copy 4 bytes user data to new buf */
memcpy(in->buf, start + 12, 4);
start = in->buf; /* point to in->buf now */
/* 12 byte skippable, so 4 bytes data done */
in->buf = start + 4;
in->size = toRead - 4;
rv = ctx->fn_read(ctx->arg_read, in);
if (rv != 0) {
pthread_mutex_unlock(&ctx->read_mutex);
return mt_error(rv);
}
if (in->size != toRead - 4)
goto error_data;
ctx->insize += in->size;
in->buf = start; /* restore inbuf */
in->size += 4;
*frame = ctx->frames++;
pthread_mutex_unlock(&ctx->read_mutex);
return 0; /* done! */
}
}
/**
* read next skippable frame (12 bytes)
* 4 bytes skippable magic
* 4 bytes little endian, must be: 4 (user data size)
* 4 bytes little endian, size to read (user data)
*/
hdr.buf = hdrbuf;
hdr.size = 12;
rv = ctx->fn_read(ctx->arg_read, &hdr);
if (rv != 0) {
pthread_mutex_unlock(&ctx->read_mutex);
return mt_error(rv);
}
/* eof reached ? */
if (unlikely(hdr.size == 0)) {
pthread_mutex_unlock(&ctx->read_mutex);
in->size = 0;
return 0;
}
/* check header data */
if (unlikely(hdr.size != 12))
goto error_read;
if (unlikely(!IsZstd_Skippable(hdr.buf)))
goto error_data;
ctx->insize += 12;
/* read new input (size should be _toRead_ bytes */
toRead = MEM_readLE32((unsigned char *)hdr.buf + 8);
{
if (in->allocated < toRead) {
/* need bigger input buffer */
if (in->allocated)
in->buf = realloc(in->buf, toRead);
else
in->buf = malloc(toRead);
if (!in->buf)
goto error_nomem;
in->allocated = toRead;
}
in->size = toRead;
rv = ctx->fn_read(ctx->arg_read, in);
if (rv != 0) {
pthread_mutex_unlock(&ctx->read_mutex);
return mt_error(rv);
}
/* needed more bytes! */
if (in->size != toRead)
goto error_data;
ctx->insize += in->size;
}
*frame = ctx->frames++;
pthread_mutex_unlock(&ctx->read_mutex);
/* done, no error */
return 0;
error_data:
pthread_mutex_unlock(&ctx->read_mutex);
return ZSTDCB_ERROR(data_error);
error_read:
pthread_mutex_unlock(&ctx->read_mutex);
return ZSTDCB_ERROR(read_fail);
error_nomem:
pthread_mutex_unlock(&ctx->read_mutex);
return ZSTDCB_ERROR(memory_allocation);
}
static void *pt_decompress(void *arg)
{
cwork_t *w = (cwork_t *) arg;
ZSTDCB_Buffer *in = &w->in;
ZSTDCB_DCtx *ctx = w->ctx;
struct writelist *wl;
size_t result = 0;
ZSTDCB_Buffer collect;
/* init dstream stream */
result = ZSTD_initDStream(w->dctx);
if (ZSTD_isError(result)) {
zstdmt_errcode = result;
return (void *)ZSTDCB_ERROR(compression_library);
}
collect.buf = 0;
collect.size = 0;
collect.allocated = 0;
for (;;) {
ZSTDCB_Buffer *out;
ZSTD_inBuffer zIn;
ZSTD_outBuffer zOut;
/* select or allocate space for new output */
pthread_mutex_lock(&ctx->write_mutex);
if (!list_empty(&ctx->writelist_free)) {
/* take unused entry */
struct list_head *entry;
entry = list_first(&ctx->writelist_free);
wl = list_entry(entry, struct writelist, node);
list_move(entry, &ctx->writelist_busy);
} else {
/* allocate new one */
wl = (struct writelist *)
malloc(sizeof(struct writelist));
if (!wl) {
result = ZSTDCB_ERROR(memory_allocation);
goto error_unlock;
}
out = &wl->out;
out->size = ctx->outputsize;
out->buf = malloc(out->size);
if (!out->buf) {
result = ZSTDCB_ERROR(memory_allocation);
goto error_unlock;
}
out->allocated = out->size;
list_add(&wl->node, &ctx->writelist_busy);
}
/* start with 512KB */
/* XXX, add framesize detection... */
out = &wl->out;
pthread_mutex_unlock(&ctx->write_mutex);
/* init dstream stream */
result = ZSTD_resetDStream(w->dctx);
if (ZSTD_isError(result)) {
zstdmt_errcode = result;
return (void *)ZSTDCB_ERROR(compression_library);
}
/* zero should not happen here! */
result = pt_read(ctx, in, &wl->frame);
if (in->size == 0)
break;
if (ZSTDCB_isError(result)) {
goto error_lock;
}
zIn.size = in->allocated;
zIn.src = in->buf;
zIn.pos = 0;
for (;;) {
again:
/* decompress loop */
zOut.size = out->allocated;
zOut.dst = out->buf;
zOut.pos = 0;
dprintf
("ZSTD_decompressStream() zIn.size=%zu zIn.pos=%zu zOut.size=%zu zOut.pos=%zu\n",
zIn.size, zIn.pos, zOut.size, zOut.pos);
result = ZSTD_decompressStream(w->dctx, &zOut, &zIn);
dprintf
("ZSTD_decompressStream(), ret=%zu zIn.size=%zu zIn.pos=%zu zOut.size=%zu zOut.pos=%zu\n",
result, zIn.size, zIn.pos, zOut.size, zOut.pos);
if (ZSTD_isError(result))
goto error_clib;
/* end of frame */
if (result == 0) {
/* put collected stuff together */
if (collect.size) {
void *bnew;
bnew = malloc(collect.size + zOut.pos);
if (!bnew) {
result =
ZSTDCB_ERROR
(memory_allocation);
goto error_lock;
}
memcpy((char *)bnew, collect.buf,
collect.size);
memcpy((char *)bnew + collect.size,
out->buf, zOut.pos);
free(collect.buf);
free(out->buf);
out->buf = bnew;
out->size = collect.size + zOut.pos;
out->allocated = out->size;
collect.buf = 0;
collect.size = 0;
} else {
out->size = zOut.pos;
}
/* write result */
pthread_mutex_lock(&ctx->write_mutex);
result = pt_write(ctx, wl);
if (ZSTDCB_isError(result))
goto error_unlock;
pthread_mutex_unlock(&ctx->write_mutex);
/* will read next input */
break;
}
/* out buffer to small for full frame */
if (result != 0) {
/* collect old content from out */
collect.buf =
realloc(collect.buf,
collect.size + out->size);
memcpy((char *)collect.buf + collect.size,
out->buf, out->size);
collect.size = collect.size + out->size;
/* double the buffer, until it fits */
pthread_mutex_lock(&ctx->write_mutex);
out->size *= 2;
ctx->outputsize = out->size;
pthread_mutex_unlock(&ctx->write_mutex);
out->buf = realloc(out->buf, out->size);
if (!out->buf) {
result =
ZSTDCB_ERROR(memory_allocation);
goto error_lock;
}
out->allocated = out->size;
goto again;
}
if (zIn.pos == zIn.size)
break; /* should fail... */
} /* decompress loop */
} /* read input loop */
/* everything is okay */
pthread_mutex_lock(&ctx->write_mutex);
list_move(&wl->node, &ctx->writelist_free);
pthread_mutex_unlock(&ctx->write_mutex);
if (in->allocated)
free(in->buf);
return 0;
error_clib:
zstdmt_errcode = result;
result = ZSTDCB_ERROR(compression_library);
/* fall through */
error_lock:
pthread_mutex_lock(&ctx->write_mutex);
error_unlock:
list_move(&wl->node, &ctx->writelist_free);
pthread_mutex_unlock(&ctx->write_mutex);
if (in->allocated)
free(in->buf);
return (void *)result;
}
/* single threaded */
static size_t st_decompress(void *arg)
{
ZSTDCB_DCtx *ctx = (ZSTDCB_DCtx *) arg;
cwork_t *w = &ctx->cwork[0];
ZSTDCB_Buffer In, Out;
ZSTDCB_Buffer *in = &In;
ZSTDCB_Buffer *out = &Out;
ZSTDCB_Buffer *magic = &w->in;
size_t result;
int rv;
ZSTD_inBuffer zIn;
ZSTD_outBuffer zOut;
/* init dstream stream */
result = ZSTD_initDStream(w->dctx);
if (ZSTD_isError(result)) {
zstdmt_errcode = result;
return ZSTDCB_ERROR(compression_library);
}
/* allocate space for input buffer */
in->size = ZSTD_DStreamInSize();
in->buf = malloc(in->size);
if (!in->buf)
return ZSTDCB_ERROR(memory_allocation);
in->allocated = in->size;
/* allocate space for output buffer */
out->size = ZSTD_DStreamOutSize();
out->buf = malloc(out->size);
if (!out->buf) {
free(in->buf);
return ZSTDCB_ERROR(memory_allocation);
}
out->allocated = out->size;
/* we read already some bytes, handle that: */
{
/* remember in->buf */
unsigned char *buf = in->buf;
/* fill first read bytes to buffer... */
memcpy(in->buf, magic->buf, magic->size);
magic->buf = in->buf;
in->buf = buf + magic->size;
in->size = in->allocated - magic->size;
/* read more bytes, to fill buffer */
rv = ctx->fn_read(ctx->arg_read, in);
if (rv != 0) {
result = mt_error(rv);
goto error;
}
/* ready, first buffer complete */
in->buf = buf;
in->size += magic->size;
ctx->insize += in->size;
}
zIn.src = in->buf;
zIn.size = in->size;
zIn.pos = 0;
zOut.dst = out->buf;
for (;;) {
for (;;) {
/* decompress loop */
zOut.size = out->allocated;
zOut.pos = 0;
result = ZSTD_decompressStream(w->dctx, &zOut, &zIn);
if (ZSTD_isError(result))
goto error_clib;
if (zOut.pos) {
ZSTDCB_Buffer wb;
wb.size = zOut.pos;
wb.buf = zOut.dst;
rv = ctx->fn_write(ctx->arg_write, &wb);
if (rv != 0) {
result = mt_error(rv);
goto error;
}
ctx->outsize += zOut.pos;
}
/* one more round */
if ((zIn.pos == zIn.size) && (result == 1) && zOut.pos)
continue;
/* finished */
if (zIn.pos == zIn.size)
break;
/* end of frame */
if (result == 0) {
result = ZSTD_resetDStream(w->dctx);
if (ZSTD_isError(result))
goto error_clib;
}
} /* decompress */
/* read next input */
in->size = in->allocated;
rv = ctx->fn_read(ctx->arg_read, in);
if (rv != 0) {
result = mt_error(rv);
goto error;
}
if (in->size == 0)
goto okay;
ctx->insize += in->size;
zIn.size = in->size;
zIn.pos = 0;
} /* read */
error_clib:
zstdmt_errcode = result;
result = ZSTDCB_ERROR(compression_library);
/* fall through */
error:
/* return with error */
free(out->buf);
free(in->buf);
return result;
okay:
/* no error */
free(out->buf);
free(in->buf);
return 0;
}
#define TYPE_UNKNOWN 0
#define TYPE_SINGLE_THREAD 1
#define TYPE_MULTI_THREAD 2
size_t ZSTDCB_decompressDCtx(ZSTDCB_DCtx * ctx, ZSTDCB_RdWr_t * rdwr)
{
unsigned char buf[16];
ZSTDCB_Buffer In;
ZSTDCB_Buffer *in = &In;
cwork_t *w;
int t, rv, type = TYPE_UNKNOWN;
void *retval_of_thread = 0;
if (!ctx)
return ZSTDCB_ERROR(compressionParameter_unsupported);
/* init reading and writing functions */
ctx->fn_read = rdwr->fn_read;
ctx->fn_write = rdwr->fn_write;
ctx->arg_read = rdwr->arg_read;
ctx->arg_write = rdwr->arg_write;
/**
* possible valid magic's for us, we need 16 bytes, for checking
*
* 1) ZSTDCB_MAGIC @0 -> ST Stream
* 2) ZSTDCB_MAGIC @0 + MAGIC_SKIPPABLE @9 -> MT Stream else ST
* 3) MAGIC_SKIPPABLE @0 + ZSTDCB_MAGIC @12 -> MT Stream
* 4) all other: not valid!
*/
/* check for ZSTDCB_MAGIC_SKIPPABLE */
in->buf = buf;
in->size = 16;
rv = ctx->fn_read(ctx->arg_read, in);
if (rv != 0)
return mt_error(rv);
/* must be single threaded standard zstd, when smaller 16 bytes */
if (in->size < 16) {
if (!IsZstd_Magic(buf))
return ZSTDCB_ERROR(data_error);
dprintf("single thread style, current pos=%zu\n", in->size);
type = TYPE_SINGLE_THREAD;
if (in->size == 9) {
/* create empty file */
ctx->threads = 0;
ctx->cwork = 0;
return 0;
}
} else {
if (IsZstd_Skippable(buf) && IsZstd_Magic(buf + 12)) {
/* pzstd */
dprintf("pzstd style\n");
type = TYPE_MULTI_THREAD;
} else if (IsZstd_Magic(buf) && IsZstd_Skippable(buf + 9)) {
/* zstdmt */
dprintf("zstdmt style\n");
type = TYPE_MULTI_THREAD;
/* set buffer to the */
} else if (IsZstd_Magic(buf)) {
/* some std zstd stream */
dprintf("single thread style, current pos=%zu\n",
in->size);
type = TYPE_SINGLE_THREAD;
} else {
/* invalid */
dprintf("not valid\n");
return ZSTDCB_ERROR(data_error);
}
}
/* use single thread extraction, when only one thread is there */
if (ctx->threadswanted == 1)
type = TYPE_SINGLE_THREAD;
/* single threaded, but with known sizes */
if (type == TYPE_SINGLE_THREAD) {
ctx->threads = 1;
ctx->cwork = (cwork_t *) malloc(sizeof(cwork_t));
if (!ctx->cwork)
return ZSTDCB_ERROR(memory_allocation);
w = &ctx->cwork[0];
w->in.buf = in->buf;
w->in.size = in->size;
w->in.allocated = 0;
w->ctx = ctx;
w->dctx = ZSTD_createDStream();
if (!w->dctx)
return ZSTDCB_ERROR(memory_allocation);
/* test, if pt_decompress is better... */
return st_decompress(ctx);
}
/* setup thread work */
ctx->threads = ctx->threadswanted;
ctx->cwork = (cwork_t *) malloc(sizeof(cwork_t) * ctx->threads);
if (!ctx->cwork)
return ZSTDCB_ERROR(memory_allocation);
for (t = 0; t < ctx->threads; t++) {
w = &ctx->cwork[t];
/* one of the threads must reuse the first bytes */
w->in.buf = in->buf;
w->in.size = in->size;
w->in.allocated = 0;
w->ctx = ctx;
w->dctx = ZSTD_createDStream();
if (!w->dctx)
return ZSTDCB_ERROR(memory_allocation);
}
/* real multi threaded, init pthread's */
pthread_mutex_init(&ctx->read_mutex, NULL);
pthread_mutex_init(&ctx->write_mutex, NULL);
pthread_mutex_init(&ctx->error_mutex, NULL);
INIT_LIST_HEAD(&ctx->writelist_free);
INIT_LIST_HEAD(&ctx->writelist_busy);
INIT_LIST_HEAD(&ctx->writelist_done);
/* multi threaded */
for (t = 0; t < ctx->threads; t++) {
cwork_t *wt = &ctx->cwork[t];
pthread_create(&wt->pthread, NULL, pt_decompress, wt);
}
/* wait for all workers */
for (t = 0; t < ctx->threads; t++) {
cwork_t *wt = &ctx->cwork[t];
void *p = 0;
pthread_join(wt->pthread, &p);
if (p)
retval_of_thread = p;
}
/* clean up pthread stuff */
pthread_mutex_destroy(&ctx->read_mutex);
pthread_mutex_destroy(&ctx->write_mutex);
pthread_mutex_destroy(&ctx->error_mutex);
/* clean up the buffers */
while (!list_empty(&ctx->writelist_free)) {
struct writelist *wl;
struct list_head *entry;
entry = list_first(&ctx->writelist_free);
wl = list_entry(entry, struct writelist, node);
free(wl->out.buf);
list_del(&wl->node);
free(wl);
}
return (size_t) retval_of_thread;
}
/* returns current uncompressed data size */
size_t ZSTDCB_GetInsizeDCtx(ZSTDCB_DCtx * ctx)
{
if (!ctx)
return 0;
return ctx->insize;
}
/* returns the current compressed data size */
size_t ZSTDCB_GetOutsizeDCtx(ZSTDCB_DCtx * ctx)
{
if (!ctx)
return 0;
return ctx->outsize;
}
/* returns the current compressed frames */
size_t ZSTDCB_GetFramesDCtx(ZSTDCB_DCtx * ctx)
{
if (!ctx)
return 0;
return ctx->curframe;
}
void ZSTDCB_freeDCtx(ZSTDCB_DCtx * ctx)
{
int t;
if (!ctx)
return;
for (t = 0; t < ctx->threads; t++) {
cwork_t *w = &ctx->cwork[t];
ZSTD_freeDStream(w->dctx);
}
if (ctx->cwork)
free(ctx->cwork);
free(ctx);
ctx = 0;
return;
}

2
CPP/7zip/Bundles/Format7zExtract/makefile
View File

@@ -199,8 +199,6 @@ ZSTDMT_OBJS = \
$O\lz4-mt_decompress.obj \
$O\lz5-mt_common.obj \
$O\lz5-mt_decompress.obj \
$O\zstd-mt_common.obj \
$O\zstd-mt_decompress.obj \
$O\zstd-mt_threading.obj \
!include "../../Aes.mak"

3
CPP/7zip/Bundles/SFXCon/makefile
View File

@@ -154,7 +154,4 @@ ZSTD_OBJS = \
$O\zstd_ldm.obj \
$O\zstd_opt.obj \
ZSTDMT_OBJS = \
$O\zstd-mt_threading.obj \
!include "../../7zip.mak"

3
CPP/7zip/Bundles/SFXWin/makefile
View File

@@ -173,7 +173,4 @@ ZSTD_OBJS = \
$O\zstd_ldm.obj \
$O\zstd_opt.obj \
ZSTDMT_OBJS = \
$O\zstd-mt_threading.obj \
!include "../../7zip.mak"