From 4c28e3fa117f9fc57544777c035ffb6ff5ed80eb Mon Sep 17 00:00:00 2001 From: Tino Reichardt Date: Sun, 20 Nov 2016 16:42:13 +0100 Subject: [PATCH] add E_ABORT detection - lz4 and lz5 are not finished... --- C/zstdmt/zstdmt.h | 7 +++++ C/zstdmt/zstdmt_compress.c | 15 ++++++++--- C/zstdmt/zstdmt_decompress.c | 19 ++++++++++++++ CPP/7zip/Compress/Lz4Decoder.cpp | 37 +++++++++++++++++++++----- CPP/7zip/Compress/Lz4Decoder.h | 5 ---- CPP/7zip/Compress/Lz4Encoder.cpp | 43 ++++++++++++++++--------------- CPP/7zip/Compress/Lz5Decoder.cpp | 40 +++++++++++++++++++++------- CPP/7zip/Compress/Lz5Decoder.h | 5 ---- CPP/7zip/Compress/Lz5Encoder.cpp | 40 ++++++++++++++-------------- CPP/7zip/Compress/ZstdDecoder.cpp | 32 +++++++++++++++++++---- CPP/7zip/Compress/ZstdDecoder.h | 5 ---- CPP/7zip/Compress/ZstdEncoder.cpp | 37 +++++++++++++------------- 12 files changed, 188 insertions(+), 97 deletions(-) diff --git a/C/zstdmt/zstdmt.h b/C/zstdmt/zstdmt.h index 114c7961..4d40f3dd 100644 --- a/C/zstdmt/zstdmt.h +++ b/C/zstdmt/zstdmt.h @@ -48,6 +48,7 @@ typedef enum { ZSTDMT_error_frame_decompress, ZSTDMT_error_compressionParameter_unsupported, ZSTDMT_error_compression_library, + ZSTDMT_error_canceled, ZSTDMT_error_maxCode } ZSTDMT_ErrorCode; @@ -73,6 +74,12 @@ typedef struct { * - you can use stdio functions or plain read/write * - just write some wrapper on your own * - a sample is given in 7-Zip ZS + * + * error definitions: + * 0 = success + * -1 = generic read/write error + * -2 = user abort + * -3 = memory */ typedef int (fn_read) (void *args, ZSTDMT_Buffer * in); typedef int (fn_write) (void *args, ZSTDMT_Buffer * out); diff --git a/C/zstdmt/zstdmt_compress.c b/C/zstdmt/zstdmt_compress.c index 48c0394f..23e828e2 100644 --- a/C/zstdmt/zstdmt_compress.c +++ b/C/zstdmt/zstdmt_compress.c @@ -15,7 +15,6 @@ #include #define ZSTD_STATIC_LINKING_ONLY -//#define DEBUGME #include "zstd.h" #include "mem.h" @@ -115,7 +114,7 @@ ZSTDMT_CCtx *ZSTDMT_createCCtx(int threads, int level, int inputsize) if (inputsize) ctx->inputsize = inputsize; else { -#ifndef DEBUGME +#if 1 const int windowLog[] = { 0, 19, 19, 20, 20, 20, 21, 21, 21, 21, 21, 22, 22, 22, 22, 22, @@ -123,7 +122,8 @@ ZSTDMT_CCtx *ZSTDMT_createCCtx(int threads, int level, int inputsize) }; ctx->inputsize = 1 << (windowLog[level] + 1); #else - const int mb[] = { 0, 1, 1, 1, 2, 2, 2, + const int mb[] = { + 0, 1, 1, 1, 2, 2, 2, 3, 3, 3, 4, 4, 4, 5, 5, 5, 5, 5, 5, 5, 5 }; @@ -180,6 +180,8 @@ static size_t pt_write(ZSTDMT_CCtx * ctx, struct writelist *wl) rv = ctx->fn_write(ctx->arg_write, &b); if (rv == -1) return ZSTDMT_ERROR(write_fail); + if (rv == -2) + return ZSTDMT_ERROR(canceled); if (b.size != 9) return ZSTDMT_ERROR(write_fail); ctx->outsize += 9; @@ -197,6 +199,8 @@ static size_t pt_write(ZSTDMT_CCtx * ctx, struct writelist *wl) rv = ctx->fn_write(ctx->arg_write, &wl->out); if (rv == -1) return ZSTDMT_ERROR(write_fail); + if (rv == -2) + return ZSTDMT_ERROR(canceled); ctx->outsize += wl->out.size; ctx->curframe++; list_move(entry, &ctx->writelist_free); @@ -264,6 +268,10 @@ static void *pt_compress(void *arg) pthread_mutex_unlock(&ctx->read_mutex); result = ZSTDMT_ERROR(read_fail); goto error; + } else if (rv == -2) { + pthread_mutex_unlock(&ctx->read_mutex); + result = ZSTDMT_ERROR(canceled); + goto error; } /* eof */ @@ -305,6 +313,7 @@ static void *pt_compress(void *arg) /* write result */ pthread_mutex_lock(&ctx->write_mutex); result = pt_write(ctx, wl); + printf("pt_write() = %d\n", result); pthread_mutex_unlock(&ctx->write_mutex); if (ZSTDMT_isError(result)) goto error; diff --git a/C/zstdmt/zstdmt_decompress.c b/C/zstdmt/zstdmt_decompress.c index ead579bb..bb45a0ca 100644 --- a/C/zstdmt/zstdmt_decompress.c +++ b/C/zstdmt/zstdmt_decompress.c @@ -175,6 +175,8 @@ static size_t pt_write(ZSTDMT_DCtx * ctx, struct writelist *wl) int rv = ctx->fn_write(ctx->arg_write, &wl->out); if (rv == -1) return ZSTDMT_ERROR(write_fail); + if (rv == -2) + return ZSTDMT_ERROR(canceled); ctx->outsize += wl->out.size; ctx->curframe++; list_move(entry, &ctx->writelist_free); @@ -217,6 +219,8 @@ static size_t pt_read(ZSTDMT_DCtx * ctx, ZSTDMT_Buffer * in, size_t * frame) rv = ctx->fn_read(ctx->arg_read, &hdr); if (rv == -1) goto error_read; + if (rv == -2) + goto error_read; if (hdr.size != 5) goto error_data; hdr.buf = hdrbuf; @@ -284,6 +288,8 @@ static size_t pt_read(ZSTDMT_DCtx * ctx, ZSTDMT_Buffer * in, size_t * frame) rv = ctx->fn_read(ctx->arg_read, &hdr); if (rv == -1) goto error_read; + if (rv == -2) + goto error_canceled; /* eof reached ? */ if (unlikely(hdr.size == 0)) { @@ -318,6 +324,8 @@ static size_t pt_read(ZSTDMT_DCtx * ctx, ZSTDMT_Buffer * in, size_t * frame) /* generic read failure! */ if (rv == -1) goto error_read; + if (rv == -2) + goto error_canceled; /* needed more bytes! */ if (in->size != toRead) goto error_data; @@ -330,6 +338,9 @@ static size_t pt_read(ZSTDMT_DCtx * ctx, ZSTDMT_Buffer * in, size_t * frame) /* done, no error */ return 0; + error_canceled: + pthread_mutex_unlock(&ctx->read_mutex); + return ZSTDMT_ERROR(canceled); error_data: pthread_mutex_unlock(&ctx->read_mutex); return ZSTDMT_ERROR(data_error); @@ -602,6 +613,14 @@ static size_t st_decompress(void *arg) w.size = zOut.pos; w.buf = zOut.dst; rv = ctx->fn_write(ctx->arg_write, &w); + if (rv == -1) { + return ZSTDMT_ERROR(write_fail); + goto error_clib; + } + if (rv == -2) { + return ZSTDMT_ERROR(canceled); + goto error_clib; + } ctx->outsize += zOut.pos; } diff --git a/CPP/7zip/Compress/Lz4Decoder.cpp b/CPP/7zip/Compress/Lz4Decoder.cpp index 85eb2a55..4af68afe 100644 --- a/CPP/7zip/Compress/Lz4Decoder.cpp +++ b/CPP/7zip/Compress/Lz4Decoder.cpp @@ -9,6 +9,16 @@ int Lz4Read(void *arg, LZ4MT_Buffer * in) size_t size = in->size; HRESULT res = ReadStream(x->inStream, in->buf, &size); + + /* catch errors */ + switch (res) { + case E_ABORT: + return -2; + case E_OUTOFMEMORY: + return -3; + } + + /* some other error -> read_fail */ if (res != S_OK) return -1; @@ -28,17 +38,29 @@ int Lz4Write(void *arg, LZ4MT_Buffer * out) { UInt32 block; HRESULT res = x->outStream->Write((char*)out->buf + done, todo, &block); + + /* catch errors */ + switch (res) { + case E_ABORT: + return -2; + case E_OUTOFMEMORY: + return -3; + } + done += block; if (res == k_My_HRESULT_WritingWasCut) break; + /* some other error -> write_fail */ if (res != S_OK) return -1; + if (block == 0) - return E_FAIL; + return -1; todo -= block; } *x->processedOut += done; + /* we need no lock here, cause only one thread can write... */ if (x->progress) x->progress->SetRatioInfo(x->processedIn, x->processedOut); @@ -130,17 +152,18 @@ HRESULT CDecoder::CodeSpec(ISequentialInStream * inStream, rdwr.arg_read = (void *)&Rd; rdwr.arg_write = (void *)&Wr; - /* 2) create compression context */ + /* 2) create decompression context */ LZ4MT_DCtx *ctx = LZ4MT_createDCtx(_numThreads, _inputSize); if (!ctx) return S_FALSE; - /* 3) compress */ - result = LZ4MT_DecompressDCtx(ctx, &rdwr); - if (result == (size_t)-LZ4MT_error_read_fail) - res = E_ABORT; - else if (LZ4MT_isError(result)) + /* 3) decompress */ + result = LZ4MT_decompressDCtx(ctx, &rdwr); + if (LZ4MT_isError(result)) { + if (result == (size_t)-LZ4MT_error_canceled) + return E_ABORT; return ErrorOut(result); + } /* 4) free resources */ LZ4MT_freeDCtx(ctx); diff --git a/CPP/7zip/Compress/Lz4Decoder.h b/CPP/7zip/Compress/Lz4Decoder.h index 0de8a2eb..7debc5aa 100644 --- a/CPP/7zip/Compress/Lz4Decoder.h +++ b/CPP/7zip/Compress/Lz4Decoder.h @@ -1,10 +1,5 @@ // (C) 2016 Tino Reichardt -/** - * you can define LZ4_LEGACY_SUPPORT to be backwards compatible (0.1 .. 0.7) - * /TR 2016-10-01 - */ - #define LZ4_STATIC_LINKING_ONLY #include "../../../C/Alloc.h" #include "../../../C/Threads.h" diff --git a/CPP/7zip/Compress/Lz4Encoder.cpp b/CPP/7zip/Compress/Lz4Encoder.cpp index 74d41b4b..f2587a74 100644 --- a/CPP/7zip/Compress/Lz4Encoder.cpp +++ b/CPP/7zip/Compress/Lz4Encoder.cpp @@ -24,6 +24,18 @@ CEncoder::~CEncoder() LZ4MT_freeCCtx(_ctx); } +HRESULT CEncoder::ErrorOut(size_t code) +{ + const char *strError = LZ4MT_getErrorString(code); + wchar_t wstrError[200+5]; /* no malloc here, /TR */ + + mbstowcs(wstrError, strError, 200); + MessageBoxW(0, wstrError, L"7-Zip ZStandard", MB_ICONERROR | MB_OK); + MyFree(wstrError); + + return S_FALSE; +} + STDMETHODIMP CEncoder::SetCoderProperties(const PROPID * propIDs, const PROPVARIANT * coderProps, UInt32 numProps) { _props.clear(); @@ -40,10 +52,11 @@ STDMETHODIMP CEncoder::SetCoderProperties(const PROPID * propIDs, const PROPVARI if (prop.vt != VT_UI4) return E_INVALIDARG; + /* level 1..22 */ _props._level = static_cast < Byte > (prop.ulVal); - Byte lz4_level = static_cast < Byte > (LZ4MT_LEVEL_MAX); - if (_props._level > lz4_level) - _props._level = lz4_level; + Byte mylevel = static_cast < Byte > (LZ4MT_LEVEL_MAX); + if (_props._level > mylevel) + _props._level = mylevel; break; } @@ -104,12 +117,12 @@ STDMETHODIMP CEncoder::Code(ISequentialInStream *inStream, return S_FALSE; /* 3) compress */ - result = LZ4MT_CompressCCtx(_ctx, &rdwr); - if (result == (size_t)-LZ4MT_error_read_fail) - res = E_ABORT; - else if (LZ4MT_isError(result)) - if (result != LZ4MT_error_read_fail) - return ErrorOut(result); + result = LZ4MT_compressCCtx(_ctx, &rdwr); + if (LZ4MT_isError(result)) { + if (result == (size_t)-LZ4MT_error_canceled) + return E_ABORT; + return ErrorOut(result); + } return res; } @@ -123,17 +136,5 @@ STDMETHODIMP CEncoder::SetNumberOfThreads(UInt32 numThreads) return S_OK; } -HRESULT CEncoder::ErrorOut(size_t code) -{ - const char *strError = LZ4MT_getErrorString(code); - wchar_t wstrError[200+5]; /* no malloc here, /TR */ - - mbstowcs(wstrError, strError, 200); - MessageBoxW(0, wstrError, L"7-Zip ZStandard", MB_ICONERROR | MB_OK); - MyFree(wstrError); - - return S_FALSE; -} - }} #endif diff --git a/CPP/7zip/Compress/Lz5Decoder.cpp b/CPP/7zip/Compress/Lz5Decoder.cpp index a5a5017f..d01b8a67 100644 --- a/CPP/7zip/Compress/Lz5Decoder.cpp +++ b/CPP/7zip/Compress/Lz5Decoder.cpp @@ -9,6 +9,16 @@ int Lz5Read(void *arg, LZ5MT_Buffer * in) size_t size = in->size; HRESULT res = ReadStream(x->inStream, in->buf, &size); + + /* catch errors */ + switch (res) { + case E_ABORT: + return -2; + case E_OUTOFMEMORY: + return -3; + } + + /* some other error -> read_fail */ if (res != S_OK) return -1; @@ -28,17 +38,29 @@ int Lz5Write(void *arg, LZ5MT_Buffer * out) { UInt32 block; HRESULT res = x->outStream->Write((char*)out->buf + done, todo, &block); + + /* catch errors */ + switch (res) { + case E_ABORT: + return -2; + case E_OUTOFMEMORY: + return -3; + } + done += block; if (res == k_My_HRESULT_WritingWasCut) break; + /* some other error -> write_fail */ if (res != S_OK) return -1; + if (block == 0) - return E_FAIL; + return -1; todo -= block; } *x->processedOut += done; + /* we need no lock here, cause only one thread can write... */ if (x->progress) x->progress->SetRatioInfo(x->processedIn, x->processedOut); @@ -130,18 +152,18 @@ HRESULT CDecoder::CodeSpec(ISequentialInStream * inStream, rdwr.arg_read = (void *)&Rd; rdwr.arg_write = (void *)&Wr; - /* 2) create compression context */ + /* 2) create decompression context */ LZ5MT_DCtx *ctx = LZ5MT_createDCtx(_numThreads, _inputSize); if (!ctx) return S_FALSE; - /* 3) compress */ - result = LZ5MT_DecompressDCtx(ctx, &rdwr); - if (result == (size_t)-LZ5MT_error_read_fail) - res = E_ABORT; - else if (LZ5MT_isError(result)) - if (result != LZ5MT_error_read_fail) - return ErrorOut(result); + /* 3) decompress */ + result = LZ5MT_decompressDCtx(ctx, &rdwr); + if (LZ5MT_isError(result)) { + if (result == (size_t)-LZ5MT_error_canceled) + return E_ABORT; + return ErrorOut(result); + } /* 4) free resources */ LZ5MT_freeDCtx(ctx); diff --git a/CPP/7zip/Compress/Lz5Decoder.h b/CPP/7zip/Compress/Lz5Decoder.h index f6c92b8f..f55b4e50 100644 --- a/CPP/7zip/Compress/Lz5Decoder.h +++ b/CPP/7zip/Compress/Lz5Decoder.h @@ -1,10 +1,5 @@ // (C) 2016 Tino Reichardt -/** - * you can define LZ5_LEGACY_SUPPORT to be backwards compatible (0.1 .. 0.7) - * /TR 2016-10-01 - */ - #define LZ5_STATIC_LINKING_ONLY #include "../../../C/Alloc.h" #include "../../../C/Threads.h" diff --git a/CPP/7zip/Compress/Lz5Encoder.cpp b/CPP/7zip/Compress/Lz5Encoder.cpp index 38e7fe52..4955db5c 100644 --- a/CPP/7zip/Compress/Lz5Encoder.cpp +++ b/CPP/7zip/Compress/Lz5Encoder.cpp @@ -24,6 +24,18 @@ CEncoder::~CEncoder() LZ5MT_freeCCtx(_ctx); } +HRESULT CEncoder::ErrorOut(size_t code) +{ + const char *strError = LZ5MT_getErrorString(code); + wchar_t wstrError[200+5]; /* no malloc here, /TR */ + + mbstowcs(wstrError, strError, 200); + MessageBoxW(0, wstrError, L"7-Zip ZStandard", MB_ICONERROR | MB_OK); + MyFree(wstrError); + + return S_FALSE; +} + STDMETHODIMP CEncoder::SetCoderProperties(const PROPID * propIDs, const PROPVARIANT * coderProps, UInt32 numProps) { _props.clear(); @@ -40,10 +52,11 @@ STDMETHODIMP CEncoder::SetCoderProperties(const PROPID * propIDs, const PROPVARI if (prop.vt != VT_UI4) return E_INVALIDARG; + /* level 1..22 */ _props._level = static_cast < Byte > (prop.ulVal); - Byte lz5_level = static_cast < Byte > (LZ5MT_LEVEL_MAX); - if (_props._level > lz5_level) - _props._level = lz5_level; + Byte mylevel = static_cast < Byte > (LZ5MT_LEVEL_MAX); + if (_props._level > mylevel) + _props._level = mylevel; break; } @@ -104,11 +117,12 @@ STDMETHODIMP CEncoder::Code(ISequentialInStream *inStream, return S_FALSE; /* 3) compress */ - result = LZ5MT_CompressCCtx(_ctx, &rdwr); - if (result == (size_t)-LZ5MT_error_read_fail) - res = E_ABORT; - if (LZ5MT_isError(result)) + result = LZ5MT_compressCCtx(_ctx, &rdwr); + if (LZ5MT_isError(result)) { + if (result == (size_t)-LZ5MT_error_canceled) + return E_ABORT; return ErrorOut(result); + } return res; } @@ -122,17 +136,5 @@ STDMETHODIMP CEncoder::SetNumberOfThreads(UInt32 numThreads) return S_OK; } -HRESULT CEncoder::ErrorOut(size_t code) -{ - const char *strError = LZ5MT_getErrorString(code); - wchar_t wstrError[200+5]; /* no malloc here, /TR */ - - mbstowcs(wstrError, strError, 200); - MessageBoxW(0, wstrError, L"7-Zip ZStandard", MB_ICONERROR | MB_OK); - MyFree(wstrError); - - return S_FALSE; -} - }} #endif diff --git a/CPP/7zip/Compress/ZstdDecoder.cpp b/CPP/7zip/Compress/ZstdDecoder.cpp index d0e7d658..cf531f62 100644 --- a/CPP/7zip/Compress/ZstdDecoder.cpp +++ b/CPP/7zip/Compress/ZstdDecoder.cpp @@ -9,6 +9,16 @@ int ZstdRead(void *arg, ZSTDMT_Buffer * in) size_t size = in->size; HRESULT res = ReadStream(x->inStream, in->buf, &size); + + /* catch errors */ + switch (res) { + case E_ABORT: + return -2; + case E_OUTOFMEMORY: + return -3; + } + + /* some other error -> read_fail */ if (res != S_OK) return -1; @@ -28,17 +38,29 @@ int ZstdWrite(void *arg, ZSTDMT_Buffer * out) { UInt32 block; HRESULT res = x->outStream->Write((char*)out->buf + done, todo, &block); + + /* catch errors */ + switch (res) { + case E_ABORT: + return -2; + case E_OUTOFMEMORY: + return -3; + } + done += block; if (res == k_My_HRESULT_WritingWasCut) break; + /* some other error -> write_fail */ if (res != S_OK) return -1; + if (block == 0) - return E_FAIL; + return -1; todo -= block; } *x->processedOut += done; + /* we need no lock here, cause only one thread can write... */ if (x->progress) x->progress->SetRatioInfo(x->processedIn, x->processedOut); @@ -137,11 +159,11 @@ HRESULT CDecoder::CodeSpec(ISequentialInStream * inStream, /* 3) decompress */ result = ZSTDMT_decompressDCtx(ctx, &rdwr); - //printf("decompress = %d / %d\n", result, ZSTDMT_error_read_fail); - if (result == (size_t)-ZSTDMT_error_read_fail) - res = E_ABORT; - else if (ZSTDMT_isError(result)) + if (ZSTDMT_isError(result)) { + if (result == (size_t)-ZSTDMT_error_canceled) + return E_ABORT; return ErrorOut(result); + } /* 4) free resources */ ZSTDMT_freeDCtx(ctx); diff --git a/CPP/7zip/Compress/ZstdDecoder.h b/CPP/7zip/Compress/ZstdDecoder.h index 394554b7..0d5c0a98 100644 --- a/CPP/7zip/Compress/ZstdDecoder.h +++ b/CPP/7zip/Compress/ZstdDecoder.h @@ -1,10 +1,5 @@ // (C) 2016 Tino Reichardt -/** - * you can define ZSTD_LEGACY_SUPPORT to be backwards compatible (0.1 .. 0.7) - * /TR 2016-10-01 - */ - #define ZSTD_STATIC_LINKING_ONLY #include "../../../C/Alloc.h" #include "../../../C/Threads.h" diff --git a/CPP/7zip/Compress/ZstdEncoder.cpp b/CPP/7zip/Compress/ZstdEncoder.cpp index 428726fc..bfdce56e 100644 --- a/CPP/7zip/Compress/ZstdEncoder.cpp +++ b/CPP/7zip/Compress/ZstdEncoder.cpp @@ -24,6 +24,18 @@ CEncoder::~CEncoder() ZSTDMT_freeCCtx(_ctx); } +HRESULT CEncoder::ErrorOut(size_t code) +{ + const char *strError = ZSTDMT_getErrorString(code); + wchar_t wstrError[200+5]; /* no malloc here, /TR */ + + mbstowcs(wstrError, strError, 200); + MessageBoxW(0, wstrError, L"7-Zip ZStandard", MB_ICONERROR | MB_OK); + MyFree(wstrError); + + return S_FALSE; +} + STDMETHODIMP CEncoder::SetCoderProperties(const PROPID * propIDs, const PROPVARIANT * coderProps, UInt32 numProps) { _props.clear(); @@ -42,9 +54,9 @@ STDMETHODIMP CEncoder::SetCoderProperties(const PROPID * propIDs, const PROPVARI /* level 1..22 */ _props._level = static_cast < Byte > (prop.ulVal); - Byte zstd_level = static_cast < Byte > (ZSTD_maxCLevel()); - if (_props._level > zstd_level) - _props._level = zstd_level; + Byte mylevel = static_cast < Byte > (ZSTDMT_LEVEL_MAX); + if (_props._level > mylevel) + _props._level = mylevel; break; } @@ -106,10 +118,11 @@ STDMETHODIMP CEncoder::Code(ISequentialInStream *inStream, /* 3) compress */ result = ZSTDMT_compressCCtx(_ctx, &rdwr); - if (result == (size_t)-ZSTDMT_error_read_fail) - res = E_ABORT; - else if (ZSTDMT_isError(result)) + if (ZSTDMT_isError(result)) { + if (result == (size_t)-ZSTDMT_error_canceled) + return E_ABORT; return ErrorOut(result); + } return res; } @@ -123,17 +136,5 @@ STDMETHODIMP CEncoder::SetNumberOfThreads(UInt32 numThreads) return S_OK; } -HRESULT CEncoder::ErrorOut(size_t code) -{ - const char *strError = ZSTDMT_getErrorString(code); - wchar_t wstrError[200+5]; /* no malloc here, /TR */ - - mbstowcs(wstrError, strError, 200); - MessageBoxW(0, wstrError, L"7-Zip ZStandard", MB_ICONERROR | MB_OK); - MyFree(wstrError); - - return S_FALSE; -} - }} #endif